Skip to content
CI

Fix multiple issues (#87,#22,#46) #138

Sign in to view logs

Fix multiple issues (#87,#22,#46)

Fix multiple issues (#87,#22,#46) #138

Workflow file for this run

.github/workflows/ci.yml at 48eb1a5
name: CI
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
on:
push:
branches:
- dev
pull_request:
release:
types: [published]
env:
NXF_ANSI_LOG: false
# URLs to Influenza ref data should be updated in step with nextflow.config
# default ncbi_influenza_fasta and ncbi_influenza_metadata params
FASTA_ZST_URL: https://api.figshare.com/v2/file/download/41415330
CSV_ZST_URL: https://api.figshare.com/v2/file/download/41415333
concurrency:
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
cancel-in-progress: true
jobs:
test_illumina:
name: Run Illumina test
# Only run on push if this is the nf-flu dev branch (merged PRs)
if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'CFIA-NCFAD/nf-flu') }}
runs-on: ubuntu-latest
env:
NXF_VER: ${{ matrix.nxf_ver }}
NXF_ANSI_LOG: false
strategy:
matrix:
# Nextflow versions: check pipeline minimum and current latest
nxf_ver: ['22.10.1', '24.04.4']
steps:
- name: Check out pipeline code
uses: actions/checkout@v4
- name: pre - df, Docker df, Docker images
if: ${{ always() }}
run: |
df -h
docker system df
docker images
- name: Cache Nextflow binary
uses: actions/cache@v4
id: cache-nextflow
with:
path: |
/usr/local/bin/nextflow
~/.nextflow
key: nextflow-${{ matrix.nxf_ver }}
- name: Install Nextflow
if: steps.cache-nextflow.outputs.cache-hit != 'true'
env:
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Cache seqtk binary
uses: actions/cache@v4
id: cache-seqtk
with:
path: /usr/local/bin/seqtk
key: seqtk
- name: Build and install seqtk
if: steps.cache-seqtk.outputs.cache-hit != 'true'
run: |
git clone --depth=1 https://github.com/lh3/seqtk.git
cd seqtk
make -j2
make install
which seqtk
- name: Cache subsampled influenza.fna
uses: actions/cache@v4
id: cache-influenza-fna
with:
path: influenza-10k.fna.zst
key: influenza-fna
- name: Subsample NCBI influenza.fna
if: steps.cache-influenza-fna.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${FASTA_ZST_URL} | zstdcat | seqtk sample -s 789 - 10000 | zstd -ck > influenza-10k.fna.zst
- name: Cache influenza.csv
uses: actions/cache@v4
id: cache-influenza-csv
with:
path: influenza.csv.zst
key: influenza-csv
- name: Download influenza.csv
if: steps.cache-influenza-csv.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${CSV_ZST_URL} > influenza.csv.zst
- name: Run pipeline with test data
run: |
nextflow run ${GITHUB_WORKSPACE} \
-profile test_illumina,docker \
--ncbi_influenza_fasta influenza-10k.fna.zst \
--ncbi_influenza_metadata influenza.csv.zst
- name: post - df, Docker df, Docker images
if: ${{ always() }}
run: |
df -h
docker system df
docker images
- name: Upload Artifact
if: success()
uses: actions/upload-artifact@v4
with:
name: illumina-test-results-${{ matrix.nxf_ver }}
path: results/pipeline_info
- name: Upload .nextflow.log
uses: actions/upload-artifact@v4
with:
name: nextflow-log-illumina-${{ matrix.nxf_ver }}
path: .nextflow.log
- name: Upload multiqc_report.html
if: success()
uses: actions/upload-artifact@v4
with:
name: illumina-test-results-multiqc-${{ matrix.nxf_ver }}
path: results/MultiQC/multiqc_report.html
test_nanopore:
name: Run Nanopore test
# Only run on push if this is the nf-flu dev branch (merged PRs)
if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'CFIA-NCFAD/nf-flu') }}
runs-on: ubuntu-latest
env:
NXF_VER: ${{ matrix.nxf_ver }}
NXF_ANSI_LOG: false
strategy:
matrix:
# Nextflow versions: check pipeline minimum and current latest
nxf_ver: ['22.10.1', '24.04.4']
steps:
- name: Check out pipeline code
uses: actions/checkout@v4
- name: pre - df, Docker df, Docker images
if: ${{ always() }}
run: |
df -h
docker system df
docker images
- name: Cache Nextflow binary
uses: actions/cache@v4
id: cache-nextflow
with:
path: |
/usr/local/bin/nextflow
~/.nextflow
key: nextflow-${{ matrix.nxf_ver }}
- name: Install Nextflow
if: steps.cache-nextflow.outputs.cache-hit != 'true'
env:
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Cache seqtk binary
uses: actions/cache@v4
id: cache-seqtk
with:
path: /usr/local/bin/seqtk
key: seqtk
- name: Build and install seqtk
if: steps.cache-seqtk.outputs.cache-hit != 'true'
run: |
git clone --depth=1 https://github.com/lh3/seqtk.git
cd seqtk
make -j2
make install
which seqtk
- name: Cache test sample reads
uses: actions/cache@v4
id: cache-reads
with:
path: reads/
key: reads
- name: Fetch IAV and IBV test seq
if: steps.cache-reads.outputs.cache-hit != 'true'
run: |
mkdir -p reads/{run1,run2}
# IBV test data
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/SRR24826962.sampled.fastq.gz > reads/SRR24826962.fastq.gz
# IAV test data
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/ERR6359501-10k.fastq.gz > reads/ERR6359501-10k.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/run1-s11-ERR6359501.fastq.gz > reads/run1/s11-ERR6359501.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/run1-s1-ERR6359501.fastq.gz > reads/run1/s1-ERR6359501.fastq.gz
# uncompressed FASTQ should work too
gunzip reads/run1/s1-ERR6359501.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/run2-s22-ERR6359501.fastq.gz > reads/run2/s22-ERR6359501.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/run2-s2-ERR6359501.fastq.gz > reads/run2/s2-ERR6359501.fastq.gz
# neg ctrl
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/ntc-bc15.fastq.gz > reads/ntc-bc15.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/ntc-bc31.fastq.gz > reads/ntc-bc31.fastq.gz
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/ntc-bc47.fastq.gz > reads/ntc-bc47.fastq.gz
- name: Prepare samplesheet.csv
run: |
echo "sample,reads" | tee -a samplesheet.csv
echo "ERR6359501-10k,$(realpath reads/ERR6359501-10k.fastq.gz)" | tee -a samplesheet.csv
echo "ERR6359501,$(realpath reads/run1)" | tee -a samplesheet.csv
echo "ERR6359501,$(realpath reads/run2)" | tee -a samplesheet.csv
echo "SRR24826962,$(realpath reads/SRR24826962.fastq.gz)" | tee -a samplesheet.csv
echo "ntc-bc15,$(realpath reads/ntc-bc15.fastq.gz)" | tee -a samplesheet.csv
echo "ntc-bc31,$(realpath reads/ntc-bc31.fastq.gz)" | tee -a samplesheet.csv
echo "ntc-bc47,$(realpath reads/ntc-bc47.fastq.gz)" | tee -a samplesheet.csv
- name: Cache subsampled influenza.fna
uses: actions/cache@v4
id: cache-influenza-fna
with:
path: influenza-10k.fna.zst
key: influenza-fna
- name: Subsample NCBI influenza.fna
if: steps.cache-influenza-fna.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${FASTA_ZST_URL} | zstdcat | seqtk sample -s 789 - 10000 | zstd -ck > influenza-10k.fna.zst
- name: Cache influenza.csv
uses: actions/cache@v4
id: cache-influenza-csv
with:
path: influenza.csv.zst
key: influenza-csv
- name: Download influenza.csv
if: steps.cache-influenza-csv.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${CSV_ZST_URL} > influenza.csv.zst
- name: Run pipeline with test data
run: |
nextflow run ${GITHUB_WORKSPACE} \
-profile test_nanopore,docker \
--platform nanopore \
--input samplesheet.csv \
--ncbi_influenza_fasta influenza-10k.fna.zst \
--ncbi_influenza_metadata influenza.csv.zst
- name: Tree of results
run: tree -h results/
- name: post - df, Docker df, Docker images
if: ${{ always() }}
run: |
df -h
docker system df
docker images
- name: Upload .nextflow.log
uses: actions/upload-artifact@v4
with:
name: nextflow-log-nanopore-${{ matrix.nxf_ver }}
path: .nextflow.log
- name: Upload pipeline_info/
if: success()
uses: actions/upload-artifact@v4
with:
name: nanopore-test-results-pipline_info-${{ matrix.nxf_ver }}
path: results/pipeline_info
- name: Upload nf-flu-subtyping-report.xlsx
if: success()
uses: actions/upload-artifact@v4
with:
name: nanopore-test-results-subtyping-report-${{ matrix.nxf_ver }}
path: results/nf-flu-subtyping-report.xlsx
- name: Upload multiqc_report.html
if: success()
uses: actions/upload-artifact@v4
with:
name: nanopore-test-results-multiqc-${{ matrix.nxf_ver }}
path: results/MultiQC/multiqc_report.html