From f457349c09101e965b5c1ddb142f8377c664d1a5 Mon Sep 17 00:00:00 2001 From: jpalmer37 Date: Tue, 6 Oct 2026 14:42:12 -0400 Subject: [PATCH 1/4] Strip a submitter prefix followed by a double dash 25-7015--WLH25-1607 cleaned to -WLH25-1607; it now cleans to WLH25-1607. Co-Authored-By: Claude Opus 5.5 --- modules/shared/sample_id.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared/sample_id.nf b/modules/shared/sample_id.nf index 86f7eca..e3a48cc 100644 --- a/modules/shared/sample_id.nf +++ b/modules/shared/sample_id.nf @@ -13,12 +13,12 @@ // kept one: the saveAs in modules/nextclade/nextclade.nf and the sample_name // column in bin/filter_alignments.py // 3. strip the submitter prefix some partner organisations prepend, -// e.g. 24-1234-IP7-SAMPLE-A -> SAMPLE-A +// e.g. 24-1234-IP7-SAMPLE-A -> SAMPLE-A, 25-7015--WLH25-1607 -> WLH25-1607 def clean_sample_id(String name) { name.replaceFirst(/(?i)_consensus$/, '') .replaceAll(/_/, '-') - .replaceFirst(/^\d{2}-\d+(?:-IP\d+)?-/, '') + .replaceFirst(/^\d{2}-\d+(?:-IP\d+)?-+/, '') } // Consensus FASTA discovery for the PAFTOOLS and NEXTCLADE branches. Each file From 7791e5e35bbf3f3f2697110dc86a7eaeca2fada3 Mon Sep 17 00:00:00 2001 From: jpalmer37 Date: Tue, 6 Oct 2026 14:42:17 -0400 Subject: [PATCH 2/4] Accept a per-sample DB bundle in the FASTA sample sheet An optional DB column gives each FASTA sample its own bundle, so one run can mix subtypes. Run-level Nextclade and reference_remap outputs move under a / directory, and aggregate/sample_ids.csv records each input ID's cleaned sample_id and bundle. Manifest version 1.1.0. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 29 ++++-- README.md | 28 ++++-- docs/database_bundles.md | 27 ++++- main.nf | 23 +++-- modules/nextclade/coordinate_conversion.nf | 15 +-- modules/nextclade/merge_flumut_mutations.nf | 8 +- modules/nextclade/nextclade.nf | 28 +++--- modules/shared/blast.nf | 9 +- modules/shared/database.nf | 25 ++++- modules/shared/provenance.nf | 6 +- modules/shared/sample_id.nf | 39 ++++++-- nextflow.config | 2 +- workflows/fasta-prep.nf | 29 ++++-- workflows/nextclade.nf | 105 +++++++++++--------- workflows/paftools.nf | 38 ++++--- workflows/short-read.nf | 9 +- 16 files changed, 270 insertions(+), 150 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e471cbb..33fe6c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,8 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and `--fastq_input`, `--fasta_input` and `--samplesheet_input` may be set. `--fastq_input` runs `SHORT_READ` and `--fasta_input` launches both `PAFTOOLS` and `NEXTCLADE`. For `--samplesheet_input` the header decides: `ID,R1,R2` runs - `SHORT_READ`, `ID,FASTA` runs the two FASTA branches. + `SHORT_READ`, `ID,FASTA` runs the two FASTA branches. An optional `DB` column + gives each FASTA sample its own bundle, so one run can mix subtypes. - `SHORT_READ` performs read QC/primer trimming, downsampling, assembly, BLAST-based reference selection, mapping, LoFreq calling, SnpEff annotation, MNV correction, consensus generation, and FluMut screening. @@ -32,13 +33,13 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and task per sample aligns each segment with minimap2, calls differences with `paftools.js`, annotates with SnpEff, converts to the common VCF shape, and corrects MNV codons. - - `NEXTCLADE` batches samples by segment, runs Nextclade once per segment, + - `NEXTCLADE` batches samples by bundle and segment, runs Nextclade once per batch, splits per-sample results, and converts JSON to the common VCF shape, annotating substitutions that share a codon with the combined change. - `--merge_flumut_mutations` is off by default. When enabled in the Nextclade - branch, a run-level map lifts coordinates from FluMut's own reference + branch, a per-bundle map lifts coordinates from FluMut's own reference (source, the `references`/`annotations` tables of the installed `flumutdb` - sqlite) onto the selected bundle's Nextclade datasets (`--db`, target), then + sqlite) onto each bundle's Nextclade datasets (target), then `bin/merge_flumut_mutations.py` adds `FLUMUT=` annotations to matching existing VCF rows. The markers and the source coordinates come from the same package, so they cannot drift apart. @@ -64,8 +65,8 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and | `docs/vcf_annotation_and_flumut_insertion.md` | Detailed Nextclade VCF and FluMutDB semantics | | `docs/watchlists.md` | Custom watchlist schema, matching rules, and the HA1/HA numbering trap | | `assets/watchlists/` | The shipped `*watchlist.csv` files; `--watchlist_dir` defaults here | -| `modules/shared/sample_id.nf` | The only place a consensus or sample sheet `sample_id` is derived; holds the FASTA discovery channel `FASTA_PREP` calls and `samplesheet_channel`, which all three workflows call for `--samplesheet_input`. Changing `clean_sample_id` renames every output | -| `modules/shared/database.nf` | The only place that knows the `--db` bundle layout; `resolve_db` validates it up front. Also holds the canonical segment list, `@Field DB_SEGMENTS`, which it uses to check each segment's dataset | +| `modules/shared/sample_id.nf` | The only place a consensus or sample sheet `sample_id` is derived; holds `fasta_sample_channel` (FASTA input with each sample's bundle), `input_db_paths` (every bundle the input names) and `samplesheet_channel` (SHORT_READ). Changing `clean_sample_id` renames every output | +| `modules/shared/database.nf` | The only place that knows the bundle layout; `resolve_db` validates one bundle up front and `resolve_dbs` every bundle a run uses, keyed by name. Also holds the canonical segment list, `@Field DB_SEGMENTS`, which it uses to check each segment's dataset | | `scripts/validate_db.py` | Deeper, contents-level bundle check; run after building or moving a bundle | | `docs/database_bundles.md` | Bundle layout, `db.yml` schema, how to build and validate one | | `README.md` | User-facing inputs, commands, outputs, and architecture | @@ -97,11 +98,16 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and underscore, and colliding pairs are dropped silently by `.unique`; sample sheet FASTQ IDs go through `clean_sample_id` instead. Avoid introducing or silently accepting colliding names. -- All reference data comes from one bundle directory, `--db`, in the fixed +- Each sample's reference data comes from one bundle directory, in the fixed layout `blastn_db.fa` + `nextclade/` + `snpeff/{snpEff.config,/}` plus an optional `db.yml`. Resolve it through `resolve_db()` in `modules/shared/database.nf` rather than composing paths at the call site; - that function is the only thing that should know the layout. + that function is the only thing that should know the layout. SHORT_READ uses + `--db`. In FASTA mode the bundle is the sample sheet's `DB` cell, else `--db`, + so a run can hold several; their names (`db.yml` `name`, else the directory) + must differ, because they key the per-bundle channels and the run-level + outputs `nextclade//`, `nextclade/status//` and + `aggregate/reference_remap//`. - The nucleotide reference FASTA used for BLAST has headers `accession|subtype|segment`. - Nextclade datasets live in uppercase per-segment subdirectories (`HA/`, `PB2/`, @@ -130,9 +136,12 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and explicitly unless a local config supplies it. - `--samplesheet_input` is parsed eagerly by `samplesheet_channel`, so column, blank-cell, ID and file-existence errors all fail before any task is submitted. - `FASTA_PREP` chooses between it and `consensus_fasta_channel`. `run_name` is the sheet's filename without `.csv`. + `fasta_sample_channel` chooses between it and `consensus_fasta_channel`, and + `FASTA_PREP` publishes `aggregate/sample_ids.csv` (`input_id,sample_id,db`), the + only record of how an input ID was cleaned. `run_name` is the sheet's filename without `.csv`. - Channel tuple keys are part of the design. Preserve `sample_id` and `segment` - through `join`, `combine`, `transpose`, and batching operations, and comment + through `join`, `combine`, `transpose`, and batching operations (and in FASTA + mode the bundle name, which keys per-bundle tasks and provenance), and comment non-obvious tuple shapes next to the transformation. - Preserve `publishDir` destinations and filenames when refactoring. Outputs are primarily `${outdir}/${sample_id}/...`, with run-level data under diff --git a/README.md b/README.md index d4d65ab..0e73743 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ flowchart TD nc2vcf[nextclade_to_freebayes_vcf.py] nc_final[Final NEXTCLADE VCFs] coord_map[coordinate_conversion: build_reference_coordinate_map.py] - coord_final[aggregate/reference_remap] + coord_final[aggregate/reference_remap, per bundle] flumut_merge[merge_flumut_mutations: merge_flumut_mutations.py] flumut_annotated[FluMutDB-annotated VCFs] @@ -140,12 +140,23 @@ SAMPLE-01,/path/to/SAMPLE-01.fasta SAMPLE-02,/path/to/SAMPLE-02.fasta ``` -runs PAFTOOLS and NEXTCLADE. Other columns are ignored. +runs PAFTOOLS and NEXTCLADE. A FASTA sheet may add a `DB` column naming each sample's +reference bundle, so one run can mix subtypes: + +```csv +ID,FASTA,DB +SAMPLE-01,/path/to/SAMPLE-01.fasta,/path/to/refs/gsgd +SAMPLE-02,/path/to/SAMPLE-02.fasta,/path/to/refs/wisconsin588 +``` + +A blank `DB` cell falls back to `--db`; see [docs/database_bundles.md](docs/database_bundles.md#several-bundles-in-one-run). Other columns are ignored. - Each `ID` is cleaned by the same rule as a FASTA filename (see the note under Usage), so `24-1234-IP7-SAMPLE_01` is reported as `SAMPLE-01`. After cleaning, an ID may contain only letters, digits, `.` and `-`. - Relative paths resolve against the directory `nextflow run` is launched from; absolute paths are safer. - The run stops before any task starts if a column or value is missing, a file does not exist, or two IDs clean to the same sample name. - `run_name`, which prefixes the run-level reports, is the sample sheet's filename without `.csv`. +- `aggregate/sample_ids.csv` maps each input ID (or FASTA filename) to its cleaned `sample_id` and bundle name. +- The short-read workflow takes a single `--db` and rejects a sheet with a `DB` column. - Give only one of `--fastq_input`, `--fasta_input` and `--samplesheet_input`. ## Reference databases @@ -289,13 +300,14 @@ The PAFTOOLS and NEXTCLADE VCFs declare and write the placeholder `FORMAT/GQ` as |------|-------------| | `${run_name}_fluscan-nf_multiqc_report.html` | MultiQC report summarising fastp, cutadapt, and FastQC results for all samples (short-read workflow only) | | `aggregate/paftools_qc_summary.tsv` | Run-level tabular summary of PAFTOOLS alignment coverage across all FASTA samples and segments | +| `aggregate/sample_ids.csv` | `input_id,sample_id,db`: each input's sample sheet `ID` (or FASTA filename), its cleaned `sample_id`, and the bundle it ran against (FASTA mode) | | `aggregate/consensus_qc_summary.tsv` | Every `*_qc_report.tsv` row from consensus preparation, concatenated across the run. A sample rejected at preparation — duplicate segment, or headers naming more than one sample — contributes no rows and is reported only in the Nextflow log | -| `aggregate/reference_remap/coordinate_map.json` | Primary machine-readable nucleotide/amino-acid coordinate map lifting flumut's reference (source) onto the selected Nextclade reference dataset (target) (NEXTCLADE workflow only) | -| `aggregate/reference_remap/coordinate_map.nt.tsv` | Flat per-position nucleotide coordinate map (source/flumut position/base -> target/nextclade position/base, per segment) | -| `aggregate/reference_remap/coordinate_map.aa.tsv` | Flat per-residue amino-acid coordinate map (source/flumut position/residue -> target/nextclade position/residue, per segment/gene) | -| `aggregate/reference_remap/coordinate_map.qc.tsv` | Per-segment/per-gene alignment QC: identity, coverage, the number of co-optimal alignments the map's gap placement was chosen from, and status (`ok`/`warning`/`missing_in_source`/`extra_in_source`/`stale_annotation`) | -| `aggregate/reference_remap/flumut_alignment.fasta` | Target/source (nextclade/flumut) nucleotide alignment blocks, one per segment | -| `aggregate/reference_remap/flumut_aa_alignment.fasta` | Target/source (nextclade/flumut) amino-acid alignment blocks, one per gene | +| `aggregate/reference_remap//coordinate_map.json` | Primary machine-readable nucleotide/amino-acid coordinate map lifting flumut's reference (source) onto bundle ``'s Nextclade reference datasets (target), one directory per bundle (NEXTCLADE workflow only) | +| `aggregate/reference_remap//coordinate_map.nt.tsv` | Flat per-position nucleotide coordinate map (source/flumut position/base -> target/nextclade position/base, per segment) | +| `aggregate/reference_remap//coordinate_map.aa.tsv` | Flat per-residue amino-acid coordinate map (source/flumut position/residue -> target/nextclade position/residue, per segment/gene) | +| `aggregate/reference_remap//coordinate_map.qc.tsv` | Per-segment/per-gene alignment QC: identity, coverage, the number of co-optimal alignments the map's gap placement was chosen from, and status (`ok`/`warning`/`missing_in_source`/`extra_in_source`/`stale_annotation`) | +| `aggregate/reference_remap//flumut_alignment.fasta` | Target/source (nextclade/flumut) nucleotide alignment blocks, one per segment | +| `aggregate/reference_remap//flumut_aa_alignment.fasta` | Target/source (nextclade/flumut) amino-acid alignment blocks, one per gene | | `${run_name}_fluscan-nf_nextflow_report.html` | Nextflow execution report | | `${run_name}_fluscan-nf_nextflow_timeline.html` | Nextflow execution timeline | | `${run_name}_fluscan-nf_nextflow_trace.txt` | Nextflow task trace | diff --git a/docs/database_bundles.md b/docs/database_bundles.md index 9856bb8..0db2276 100644 --- a/docs/database_bundles.md +++ b/docs/database_bundles.md @@ -1,8 +1,10 @@ # Reference database bundles Every reference the pipeline needs for one reference set lives in a single directory — a -*bundle* — passed as `--db`. The FluMut coordinate reference used by the optional merge -is not a bundle: it is read from the `flumutdb` conda package. +*bundle* — passed as `--db`. In FASTA mode a sample sheet's `DB` column can instead give +each sample its own bundle (see [Several bundles in one run](#several-bundles-in-one-run)). +The FluMut coordinate reference used by the optional merge is not a bundle: it is read from +the `flumutdb` conda package. ``` nextflow run BCCDC-PHL/fluscan-nf \ @@ -34,7 +36,24 @@ nests them one level deeper under `data/`; a bundle does not, because `snpeff/` already the per-bundle directory that level provided. The filenames are fixed and not configurable. `modules/shared/database.nf` is the only -place that knows this layout; `--db` is the only path you supply. +place that knows this layout; the bundle root is the only path you supply. + +### Several bundles in one run + +In FASTA mode a sample sheet may carry a `DB` column holding a bundle root per row: + +```csv +ID,FASTA,DB +SAMPLE-01,/path/to/SAMPLE-01.fasta,/path/to/refs/gsgd +SAMPLE-02,/path/to/SAMPLE-02.fasta,/path/to/refs/wisconsin588 +``` + +A blank `DB` cell, or a sheet without the column, falls back to `--db`. Every bundle is +validated before any task runs, and each sample is analysed against its own bundle only. +Bundle names (`db.yml` `name`, else the directory name) must differ within a run, because +run-level outputs are filed under them: `nextclade//`, `nextclade/status//` +and `aggregate/reference_remap//`. The short-read workflow takes one `--db` and +rejects a sheet with a `DB` column. ### Bundles may be partial @@ -74,7 +93,7 @@ Quote `version`. Unquoted `2025-07-07` parses as a *date*, not a string, in both YAML reader the pipeline uses and the one that reads the provenance file back. `nextclade_ref_name` is what appears as `target_reference` in -`aggregate/reference_remap/coordinate_map.json`. It is separate from `name` so a bundle +`aggregate/reference_remap//coordinate_map.json`. It is separate from `name` so a bundle can be renamed or relocated on disk without changing labels already written into published coordinate maps. diff --git a/main.nf b/main.nf index 7728de7..aa20410 100644 --- a/main.nf +++ b/main.nf @@ -9,13 +9,14 @@ include { SHORT_READ } from './workflows/short-read.nf' include { FASTA_PREP } from './workflows/fasta-prep.nf' include { PAFTOOLS } from './workflows/paftools.nf' include { NEXTCLADE } from './workflows/nextclade.nf' -include { resolve_db } from './modules/shared/database.nf' -include { db_summary } from './modules/shared/database.nf' +include { resolve_dbs } from './modules/shared/database.nf' +include { db_summary } from './modules/shared/database.nf' +include { input_db_paths } from './modules/shared/sample_id.nf' -// Resolved with no required components: this only checks that --db points at something -// that exists, so a typo fails before the banner. Each workflow re-resolves with the -// components it actually consumes. -def db = resolve_db(params.db, []) +// Resolved with no required components: this only checks that every bundle the input names +// exists, so a typo fails before the banner. Each workflow re-resolves with the components +// it actually consumes. +def dbs = resolve_dbs(input_db_paths(), []) // prints to the screen and to the log log.info """ @@ -26,7 +27,7 @@ log.info """ fastqInputDir : ${params.fastq_input} fastaInputDir : ${params.fasta_input} samplesheet : ${params.samplesheet_input} - database : ${db_summary(db)} + database : ${dbs.values().collect { db_summary(it) }.join('\n ')} primers : ${params.primers} outdir : ${params.outdir} pipeline run name : ${workflow.runName} @@ -50,12 +51,16 @@ workflow { ? (file(params.samplesheet_input, checkIfExists: true).splitCsv(strip: true)[0] ?: []) : [] + if ('R1' in header && 'DB' in header) { + error "ERROR: the short-read workflow takes one --db bundle; remove the DB column from ${params.samplesheet_input}." + } + if (params.fastq_input != 'NO_FILE' || 'R1' in header) { SHORT_READ() } else if (params.fasta_input != 'NO_FILE' || 'FASTA' in header) { FASTA_PREP() - PAFTOOLS(FASTA_PREP.out.cleaned, FASTA_PREP.out.provenance) - NEXTCLADE(FASTA_PREP.out.cleaned, FASTA_PREP.out.provenance) + PAFTOOLS(FASTA_PREP.out.cleaned, FASTA_PREP.out.db, FASTA_PREP.out.provenance) + NEXTCLADE(FASTA_PREP.out.cleaned, FASTA_PREP.out.db, FASTA_PREP.out.provenance) } else if (params.samplesheet_input != 'NO_FILE') { error "ERROR: --samplesheet_input needs an ID,R1,R2 header (short-read workflow) or an ID,FASTA header (PAFTOOLS/NEXTCLADE workflow)." } else { diff --git a/modules/nextclade/coordinate_conversion.nf b/modules/nextclade/coordinate_conversion.nf index 7223a9e..33f4754 100644 --- a/modules/nextclade/coordinate_conversion.nf +++ b/modules/nextclade/coordinate_conversion.nf @@ -1,6 +1,6 @@ // nextclade/coordinate_conversion.nf - Build a FluMut->Nextclade reference coordinate map. // -// Runs once per pipeline invocation, not per sample. Translates nucleotide and amino-acid +// Runs once per bundle, not per sample. Translates nucleotide and amino-acid // indices from flumut's fixed reference (source, read from the installed flumutdb package) // onto the selected Nextclade dataset (target), so merge_flumut_mutations can place FluMutDB // positions in Nextclade coordinates. Idempotent: the same flumutdb and dataset always give @@ -12,16 +12,17 @@ process coordinate_conversion { conda "${projectDir}/environments/fluscan.yml" - publishDir "${params.outdir}/aggregate", pattern: "reference_remap", mode: 'copy' + publishDir "${params.outdir}/aggregate/reference_remap", pattern: "reference_remap", mode: 'copy', + saveAs: { db_name } input: - tuple val(target_name), path(target_dataset_dir, stageAs: 'target_dataset') + tuple val(db_name), val(target_name), path(target_dataset_dir, stageAs: 'target_dataset') output: - path("reference_remap"), emit: map_dir - path("reference_remap/coordinate_map.json"), emit: json - path("reference_remap/coordinate_map.*.json"), emit: json_shards - path("reference_remap/coordinate_conversion_provenance.yml"), emit: provenance + tuple val(db_name), path("reference_remap"), emit: map_dir + tuple val(db_name), path("reference_remap/coordinate_map.json"), emit: json + tuple val(db_name), path("reference_remap/coordinate_map.*.json"), emit: json_shards + tuple val(db_name), path("reference_remap/coordinate_conversion_provenance.yml"), emit: provenance script: """ diff --git a/modules/nextclade/merge_flumut_mutations.nf b/modules/nextclade/merge_flumut_mutations.nf index 13b3df0..da9ab7c 100644 --- a/modules/nextclade/merge_flumut_mutations.nf +++ b/modules/nextclade/merge_flumut_mutations.nf @@ -7,7 +7,7 @@ process merge_flumut_mutations { - tag { segment } + tag { "${db_name}/${segment}" } conda "${projectDir}/environments/fluscan.yml" @@ -27,11 +27,11 @@ process merge_flumut_mutations { } input: - tuple val(segment), path(vcfs, stageAs: 'vcf/*'), path(coord_json) + tuple val(db_name), val(segment), path(vcfs, stageAs: 'vcf/*'), path(coord_json) output: - tuple val(segment), path("*_flumut_annotated.vcf"), emit: vcf - tuple val(segment), path("${segment}_merge_flumut_provenance.yml"), emit: provenance + tuple val(db_name), val(segment), path("*_flumut_annotated.vcf"), emit: vcf + tuple val(db_name), val(segment), path("${segment}_merge_flumut_provenance.yml"), emit: provenance script: """ diff --git a/modules/nextclade/nextclade.nf b/modules/nextclade/nextclade.nf index a5ae091..4103f1b 100644 --- a/modules/nextclade/nextclade.nf +++ b/modules/nextclade/nextclade.nf @@ -1,14 +1,14 @@ // nextclade/nextclade.nf - Per-segment Nextclade alignment and VCF conversion processes -// Cross-sample per-segment batch: runs nextclade once for all samples' sequences for a -// given segment, splits the output into per-sample JSON, TSV and identity QC files, and +// Cross-sample per-segment batch: runs nextclade once for one bundle's samples' sequences +// for a given segment, splits the output into per-sample JSON, TSV and identity QC files, and // converts it to one final VCF per sample. A sample Nextclade cannot align gets no JSON and // no VCF, and an identity QC row of FAIL. process nextclade_run_batch { conda "${projectDir}/environments/nextclade.yml" - tag { segment } + tag { "${db_name}/${segment}" } publishDir "${params.outdir}", mode: 'copy', saveAs: { filename -> @@ -24,7 +24,7 @@ process nextclade_run_batch { } if (filename =~ /.*_nextclade_status\.tsv$/) { def workflow = task.ext.workflow ?: 'unknown_workflow' - return "${workflow}/status/${filename}" + return "${workflow}/status/${db_name}/${filename}" } if (filename =~ /.*_nextclade\.(json|tsv)$/) { def file_extension = filename.endsWith('.json') ? 'json' : 'tsv' @@ -35,23 +35,23 @@ process nextclade_run_batch { } if (filename =~ /nc_output.*/) { def newname = filename.replaceAll(/nc_output/, "${segment}_output") - return "nextclade/${newname}" + return "nextclade/${db_name}/${newname}" } return null } input: - tuple val(segment), path(batch_fasta, stageAs: 'batch.fasta'), path(dataset_dir) + tuple val(db_name), val(segment), path(batch_fasta, stageAs: 'batch.fasta'), path(dataset_dir) output: - tuple val(segment), path("*_nc_final.vcf"), optional: true, emit: vcf - tuple val(segment), path("*_nc_nomat_final.vcf"), optional: true, emit: vcf_no_mature - tuple val(segment), path("*_nextclade.json"), optional: true, emit: json - tuple val(segment), path("*_nextclade.tsv"), emit: tsv - tuple val(segment), path("*_identity_qc.tsv"), emit: identity_qc - tuple val(segment), path("nc_output"), emit: all - tuple val(segment), path("${segment}_nextclade_status.tsv"), emit: status - tuple val(segment), path("${segment}_nextclade_provenance.yml"), emit: provenance + tuple val(db_name), val(segment), path("*_nc_final.vcf"), optional: true, emit: vcf + tuple val(db_name), val(segment), path("*_nc_nomat_final.vcf"), optional: true, emit: vcf_no_mature + tuple val(db_name), val(segment), path("*_nextclade.json"), optional: true, emit: json + tuple val(db_name), val(segment), path("*_nextclade.tsv"), emit: tsv + tuple val(db_name), val(segment), path("*_identity_qc.tsv"), emit: identity_qc + tuple val(db_name), val(segment), path("nc_output"), emit: all + tuple val(db_name), val(segment), path("${segment}_nextclade_status.tsv"), emit: status + tuple val(db_name), val(segment), path("${segment}_nextclade_provenance.yml"), emit: provenance script: def threads = task.cpus ?: 1 diff --git a/modules/shared/blast.nf b/modules/shared/blast.nf index 93eb6b8..1b62114 100644 --- a/modules/shared/blast.nf +++ b/modules/shared/blast.nf @@ -2,15 +2,15 @@ process makeblastdb { - tag { fasta_db.name } + tag { db_name } conda "${projectDir}/environments/fluscan.yml" input: - path(fasta_db) + tuple val(db_name), path(fasta_db) output: - path("${blast_db_name}*"), emit: db + tuple val(db_name), path("${blast_db_name}*"), emit: db script: blast_db_name = "${task.ext.workflow}_blastn_db.fasta" @@ -36,8 +36,7 @@ process blastn { publishDir "${params.outdir}/${sample_id}/", pattern: "${sample_id}*fasta" , mode:'copy' input: - tuple val(sample_id), path(query) - path(blast_db) + tuple val(sample_id), path(query), path(blast_db) output: path("${sample_id}*.tsv") // declared so publishDir sees them; nothing reads them diff --git a/modules/shared/database.nf b/modules/shared/database.nf index 0318e03..04d2e56 100644 --- a/modules/shared/database.nf +++ b/modules/shared/database.nf @@ -16,13 +16,16 @@ // does not, because snpeff/ is already the per-bundle directory that level provided. // // The filenames are fixed on purpose. A bundle is identified by its layout, so this -// file is the only place that knows where anything inside one lives; --db is the only -// path the caller supplies. +// file is the only place that knows where anything inside one lives; the bundle root is +// the only path the caller supplies. // // Bundles may be partial. Each workflow declares the components it consumes and only // those are validated: SHORT_READ requires 'blastn' (and picks up snpeff annotation // when the bundle happens to carry it), PAFTOOLS requires 'blastn' and 'snpeff', // NEXTCLADE requires 'nextclade'. +// +// SHORT_READ uses the one --db bundle. In FASTA mode each sample carries its own bundle +// (the sample sheet's DB column, else --db), so one run can hold several. import org.yaml.snakeyaml.Yaml @@ -119,15 +122,15 @@ def built_snpeff_genomes(snpeff_dir) { def resolve_db(String path, List required) { if (!path || path == 'NO_FILE') { - error "ERROR: --db must be set to a reference database bundle.\n" + db_layout_help() + error "ERROR: --db (or a sample sheet DB value) must be set to a reference database bundle.\n" + db_layout_help() } def root = file(path) if (!root.exists()) { - error "ERROR: --db ${path} does not exist." + error "ERROR: reference database bundle ${path} does not exist." } if (!root.isDirectory()) { - error "ERROR: --db ${path} is not a directory; it must be a bundle root.\n" + db_layout_help() + error "ERROR: reference database bundle ${path} is not a directory; it must be a bundle root.\n" + db_layout_help() } def blastn = root.resolve('blastn_db.fa') @@ -215,6 +218,18 @@ def resolve_db(String path, List required) { } +// Resolve every bundle in `paths` with resolve_db, keyed by bundle name. The name labels +// run-level outputs, so two different bundles may not share one. +def resolve_dbs(List paths, List required) { + def dbs = paths.collect { resolve_db(it, required) }.unique { it.root } + def clashes = dbs.countBy { it.name }.findAll { it.value > 1 }.keySet() + if (clashes) { + error "ERROR: different bundles share the name ${clashes.join(', ')}; give each a distinct name in its db.yml." + } + return dbs.collectEntries { [(it.name): it] } +} + + def db_summary(Map db) { def components = [ db.has_blastn ? 'blastn' : null, diff --git a/modules/shared/provenance.nf b/modules/shared/provenance.nf index 5bf8263..8b96410 100644 --- a/modules/shared/provenance.nf +++ b/modules/shared/provenance.nf @@ -25,15 +25,15 @@ process collect_provenance { process pipeline_provenance { - tag { pipeline_name + " / " + pipeline_version } + tag { db_name } executor 'local' input: - tuple val(session_id), val(run_name), val(pipeline_name), val(pipeline_version), val(timestamp_analysis_start), val(databases) + tuple val(db_name), val(session_id), val(run_name), val(pipeline_name), val(pipeline_version), val(timestamp_analysis_start), val(databases) output: - file("pipeline_provenance.yml") + tuple val(db_name), path("pipeline_provenance.yml") script: // `databases` is an insertion-ordered map of reference-database facts, so which keys diff --git a/modules/shared/sample_id.nf b/modules/shared/sample_id.nf index e3a48cc..8bf53be 100644 --- a/modules/shared/sample_id.nf +++ b/modules/shared/sample_id.nf @@ -40,10 +40,10 @@ def consensus_fasta_channel(String input_dir) { return ch } -// Sample sheet discovery for every workflow: an ID column plus the given file columns, -// ['R1', 'R2'] or ['FASTA']. Parsed eagerly, so a bad sheet fails before any task runs. -// Relative paths resolve against the launch directory. -def samplesheet_channel(String path, List columns) { +// Validated sample sheet rows as [sample_id, row] pairs: an ID column plus the given file +// columns, ['R1', 'R2'] or ['FASTA']. Parsed eagerly, so a bad sheet fails before any task +// runs. +def samplesheet_rows(String path, List columns) { def rows = file(path, checkIfExists: true).splitCsv(header: true, strip: true) if (!rows) { error "ERROR: sample sheet ${path} has no rows." @@ -71,8 +71,35 @@ def samplesheet_channel(String path, List columns) { error "ERROR: IDs in ${path} collapse to duplicate sample IDs: ${duplicates.join(', ')}" } - // [sample_id, file, ...] in the order of columns, e.g. [sample_id, R1, R2] - return Channel.fromList([ids, rows].transpose().collect { id, row -> + return [ids, rows].transpose() +} + +// [sample_id, file, ...] in the order of columns, e.g. [sample_id, R1, R2]. Relative paths +// resolve against the launch directory. +def samplesheet_channel(String path, List columns) { + return Channel.fromList(samplesheet_rows(path, columns).collect { id, row -> [id] + columns.collect { file(row[it], checkIfExists: true) } }) } + +// FASTA-mode samples as [sample_id, FASTA, bundle path, input ID]. The bundle is the sample +// sheet's DB cell, or --db for a blank cell, a sheet without DB, and --fasta_input. +def fasta_sample_channel() { + if (params.samplesheet_input == 'NO_FILE') { + return consensus_fasta_channel(params.fasta_input) + .map { id, fasta -> [id, fasta, params.db, fasta.name] } + } + return Channel.fromList(samplesheet_rows(params.samplesheet_input, ['FASTA']).collect { id, row -> + [id, file(row.FASTA, checkIfExists: true), row.DB ?: params.db, row.ID] + }) +} + +// Every bundle path the input names, by the same rule as fasta_sample_channel. Read eagerly +// so each bundle is validated before any task runs. +def input_db_paths() { + def rows = params.samplesheet_input == 'NO_FILE' ? [] : file(params.samplesheet_input).splitCsv(header: true, strip: true) + if (!rows || !('FASTA' in rows[0].keySet())) { + return [params.db] + } + return rows.collect { row -> row.DB ?: params.db }.unique() +} diff --git a/nextflow.config b/nextflow.config index 82e011d..d463f28 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,7 @@ manifest { author = 'John Palmer' name = 'BCCDC-PHL/fluscan-nf' - version = '0.1.0' + version = '1.1.0' description = 'BCCDC-PHL FluScan' mainScript = 'main.nf' nextflowVersion = '>=22.10.0' diff --git a/workflows/fasta-prep.nf b/workflows/fasta-prep.nf index 47b6166..de2ac0a 100644 --- a/workflows/fasta-prep.nf +++ b/workflows/fasta-prep.nf @@ -18,18 +18,32 @@ include { hash_files } from '../modules/shared/hash_files.nf' include { prepare_consensus } from '../modules/shared/consensus_qc.nf' include { flumut } from '../modules/shared/flumut.nf' -include { consensus_fasta_channel } from '../modules/shared/sample_id.nf' -include { samplesheet_channel } from '../modules/shared/sample_id.nf' +include { fasta_sample_channel } from '../modules/shared/sample_id.nf' +include { input_db_paths } from '../modules/shared/sample_id.nf' +include { resolve_dbs } from '../modules/shared/database.nf' workflow FASTA_PREP { main: - // Consensus FASTAs from the sample sheet or the input directory. - // Each file is expected to contain 8 segments for a single sample. - ch_fasta_input = params.samplesheet_input != 'NO_FILE' - ? samplesheet_channel(params.samplesheet_input, ['FASTA']) - : consensus_fasta_channel(params.fasta_input) + // Consensus FASTAs from the sample sheet or the input directory, with each sample's + // bundle. Each file is expected to contain 8 segments for a single sample. + def dbs = resolve_dbs(input_db_paths(), []) + ch_samples = fasta_sample_channel() // [sample_id, FASTA, bundle path, input ID] + ch_fasta_input = ch_samples.map { sample_id, fasta, db_path, input_id -> [sample_id, fasta] } + ch_db = ch_samples.map { sample_id, fasta, db_path, input_id -> + [sample_id, dbs.values().find { it.root == file(db_path) }.name] + } + + // Input ID to sample_id and bundle, for joining results back to the submitter's IDs. + ch_samples.join(ch_db) + .map { sample_id, fasta, db_path, input_id, db_name -> "${input_id},${sample_id},${db_name}\n" } + .collectFile( + name: 'sample_ids.csv', + storeDir: "${params.outdir}/aggregate", + seed: "input_id,sample_id,db\n", + sort: true + ) // Hash the input FASTA. One file per sample, so no list-wrapping here. hash_files(ch_fasta_input.combine(Channel.of("fasta_input"))) @@ -60,5 +74,6 @@ workflow FASTA_PREP { emit: cleaned = prepare_consensus.out.cleaned // [sample_id, cleaned consensus FASTA] + db = ch_db // [sample_id, bundle name] provenance = ch_provenance // [sample_id, [hash, prepare, flumut]] } diff --git a/workflows/nextclade.nf b/workflows/nextclade.nf index 4a88f07..9f83a35 100644 --- a/workflows/nextclade.nf +++ b/workflows/nextclade.nf @@ -4,7 +4,7 @@ * Nextclade consensus FASTA workflow * ------------------------------------ * Takes the consensus FASTAs FASTA_PREP has cleaned through per-segment Nextclade - * alignment against the --db bundle's datasets, and conversion of the JSON to a + * alignment against each sample's bundle's datasets, and conversion of the JSON to a * freebayes-style VCF. */ @@ -14,20 +14,23 @@ include { merge_flumut_mutations } from '../modules/nextclade/merg include { mutation_watch } from '../modules/shared/mutation_watch.nf' include { pipeline_provenance } from '../modules/shared/provenance.nf' include { collect_provenance } from '../modules/shared/provenance.nf' -include { resolve_db } from '../modules/shared/database.nf' +include { resolve_dbs } from '../modules/shared/database.nf' +include { input_db_paths } from '../modules/shared/sample_id.nf' workflow NEXTCLADE { take: ch_cleaned // [sample_id, cleaned consensus FASTA] + ch_db // [sample_id, bundle name] ch_prep_provenance // [sample_id, [hash, prepare, flumut]] main: // Only the Nextclade datasets are read here; this branch never BLASTs or runs SnpEff. - def db = resolve_db(params.db, ['nextclade']) + def dbs = resolve_dbs(input_db_paths(), ['nextclade']) - ch_workflow_metadata = Channel.value([ + ch_workflow_metadata = Channel.fromList(dbs.values().collect { db -> [ + db.name, workflow.sessionId, workflow.runName, workflow.manifest.name, @@ -38,21 +41,24 @@ workflow NEXTCLADE { database_version: db.version, database_path: db.root, ], - ]) + ] }) - ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata) + ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata) // [bundle name, record] - // --- Step 1: Collect each segment across all samples into one FASTA --- + // --- Step 1: Collect each segment across one bundle's samples into one FASTA --- // Cleaned headers are {sample_id}_{SEGMENT} with a canonical SEGMENT; the batch // header is the bare sample_id, which nextclade reports back as seqName. - ch_segment_batches = ch_cleaned - .splitFasta(elem: 1, record: [id: true, seqString: true]) // [sample_id, record] + ch_segment_batches = ch_cleaned.join(ch_db) + .splitFasta(elem: 1, record: [id: true, seqString: true]) // [sample_id, record, bundle name] .collectFile { item -> - def (sample_id, record) = item - ["${record.id - "${sample_id}_"}.fasta", ">${sample_id}\n${record.seqString}\n"] + def (sample_id, record, db_name) = item + ["${db_name}__${record.id - "${sample_id}_"}.fasta", ">${sample_id}\n${record.seqString}\n"] } - .map { fasta -> [fasta.baseName, fasta, db.nextclade.resolve(fasta.baseName)] } - // -> [segment, batch.fasta, dataset_dir] + .map { fasta -> + def (db_name, segment) = (fasta.baseName =~ /^(.+)__([^_]+)$/)[0][1..2] + [db_name, segment, fasta, dbs[db_name].nextclade.resolve(segment)] + } + // -> [bundle name, segment, batch.fasta, dataset_dir] // --- Step 2: Run nextclade once per segment for all samples, and convert to VCF --- // Also writes each sample's report-only consensus/reference identity QC. @@ -70,10 +76,10 @@ workflow NEXTCLADE { ch_watchlists = Channel.fromPath( params.watchlist_dir + "/*watchlist.csv", checkIfExists: true ) - // [segment, [vcfs]] -> [sample_id, [segments], [vcfs]]: one scan task per sample. + // [bundle, segment, [vcfs]] -> [sample_id, [segments], [vcfs]]: one scan task per sample. ch_sample_vcfs = nextclade_run_batch.out.vcf .transpose() - .map { segment, vcf -> [vcf.name.split('__')[0], segment, vcf] } + .map { db_name, segment, vcf -> [vcf.name.split('__')[0], segment, vcf] } .groupTuple() mutation_watch(ch_sample_vcfs, ch_watchlists.collect()) } @@ -83,69 +89,72 @@ workflow NEXTCLADE { // stops at the *_final.vcf produced above. if (params.merge_flumut_mutations) { - // --- Step 4: Build the flumut-to-Nextclade coordinate map --- - // source = the flumutdb package's reference (lift FROM), target = the --db + // --- Step 4: Build the flumut-to-Nextclade coordinate map, once per bundle --- + // source = the flumutdb package's reference (lift FROM), target = the bundle's // datasets (lift ONTO). The target label comes from the bundle's db.yml, so the // published map names the reference rather than where the bundle sits. - coordinate_conversion(Channel.value([db.nextclade_ref_name, db.nextclade])) + coordinate_conversion(Channel.fromList(dbs.values().collect { db -> [db.name, db.nextclade_ref_name, db.nextclade] })) // --- Step 5: Annotate every sample's VCFs with FluMutDB markers, one task per segment --- // Additive: writes a separate *_flumut_annotated.vcf and leaves *_final.vcf // untouched. Each task stages only its own segment's shard. - // Key each per-segment shard by its segment label parsed from the filename. + // Key each per-segment shard by its bundle and the segment label parsed from the filename. ch_shards = coordinate_conversion.out.json_shards - .flatten() - .map { f -> // [SEG, shard] + .transpose() + .map { db_name, f -> // [bundle, SEG, shard] def matcher = f.name =~ /coordinate_map\.([^.]+)\.json/ if (!matcher) { error "coordinate_conversion emitted '${f.name}', which does not follow " + "coordinate_map..json; cannot key it by segment" } - [matcher[0][1], f] + [db_name, matcher[0][1], f] } - // [segment, [vcfs]] + shard -> [segment, [vcfs], coord_shard] - merge_flumut_mutations(nextclade_run_batch.out.vcf.combine(ch_shards, by: 0)) + // [bundle, segment, [vcfs]] + shard -> [bundle, segment, [vcfs], coord_shard] + merge_flumut_mutations(nextclade_run_batch.out.vcf.combine(ch_shards, by: [0, 1])) } - // Per-sample provenance. Nextclade runs once per segment, so every segment's record - // is combined onto every sample. + // Per-sample provenance, keyed [bundle, sample_id, records] until the end. Nextclade + // runs once per bundle and segment, so each segment's record is combined onto every + // sample of that bundle. ch_nextclade_provenance = nextclade_run_batch.out.provenance - .map { segment, record -> record } - .collect() - .map { records -> [records] } - ch_provenance = ch_prep_provenance - .combine(ch_pipeline_provenance) - .combine(ch_nextclade_provenance) - .map{ sample_id, records, pipeline, nextclade -> [sample_id, [pipeline] + records + nextclade] } + .map { db_name, segment, record -> [db_name, record] } + .groupTuple() + ch_provenance = ch_prep_provenance.join(ch_db) + .map { sample_id, records, db_name -> [db_name, sample_id, records] } + .combine(ch_pipeline_provenance, by: 0) + .combine(ch_nextclade_provenance, by: 0) + .map { db_name, sample_id, records, pipeline, nextclade -> [db_name, sample_id, [pipeline] + records + nextclade] } if (params.watchlist_dir != 'NO_FILE') { // One scan task per sample. remainder: true keeps samples whose scan failed. ch_provenance = ch_provenance + .map { db_name, sample_id, records -> [sample_id, records] } .join(mutation_watch.out.provenance, remainder: true) - .map{ sample_id, records, watch -> [sample_id, (records ?: []) + (watch ? [watch] : [])] } + .map { sample_id, records, watch -> [sample_id, (records ?: []) + (watch ? [watch] : [])] } + .join(ch_db) + .map { sample_id, records, db_name -> [db_name, sample_id, records] } } if (params.merge_flumut_mutations) { - // One merge task per segment, so every segment's record is combined onto every - // sample. ifEmpty keeps the chain alive when no segment produced a VCF. + // One merge task per bundle and segment, so each segment's record is combined onto + // every sample of that bundle. A bundle with no VCF to merge gets an empty list. ch_merge_provenance = merge_flumut_mutations.out.provenance - .map { segment, record -> record } - .collect() - .ifEmpty([]) - .map { records -> [records] } - ch_provenance = ch_provenance - .combine(ch_merge_provenance) - .map{ sample_id, records, merge -> [sample_id, records + merge] } + .map { db_name, segment, record -> [db_name, record] } + .groupTuple() + ch_merge_provenance = Channel.fromList(dbs.keySet().collect { [it] }) + .join(ch_merge_provenance, remainder: true) + .map { db_name, records -> [db_name, records ?: []] } - // coordinate_conversion runs once per pipeline invocation, not per sample, so - // its single provenance file is combined onto every sample rather than joined. + // coordinate_conversion runs once per bundle, not per sample, so its provenance + // file is combined onto every sample of that bundle. ch_provenance = ch_provenance - .combine(coordinate_conversion.out.provenance) - .map{ it -> [it[0], it[1] << it[2]] } + .combine(ch_merge_provenance, by: 0) + .combine(coordinate_conversion.out.provenance, by: 0) + .map { db_name, sample_id, records, merge, coord -> [db_name, sample_id, records + merge + [coord]] } } - collect_provenance(ch_provenance) + collect_provenance(ch_provenance.map { db_name, sample_id, records -> [sample_id, records] }) } diff --git a/workflows/paftools.nf b/workflows/paftools.nf index ab413e5..c8a983e 100644 --- a/workflows/paftools.nf +++ b/workflows/paftools.nf @@ -20,21 +20,24 @@ include { makeblastdb } from '../modules/shared/blast.nf' include { blastn } from '../modules/shared/blast.nf' include { paftools_variants } from '../modules/paftools/paftools.nf' include { mutation_watch } from '../modules/shared/mutation_watch.nf' -include { resolve_db } from '../modules/shared/database.nf' +include { resolve_dbs } from '../modules/shared/database.nf' +include { input_db_paths } from '../modules/shared/sample_id.nf' workflow PAFTOOLS { take: ch_cleaned // [sample_id, cleaned consensus FASTA] + ch_db // [sample_id, bundle name] ch_prep_provenance // [sample_id, [hash, prepare, flumut]] main: // Reference selection is BLAST-driven and every called variant is SnpEff-annotated, // so both components are mandatory for this branch. - def db = resolve_db(params.db, ['blastn', 'snpeff']) + def dbs = resolve_dbs(input_db_paths(), ['blastn', 'snpeff']) - ch_workflow_metadata = Channel.value([ + ch_workflow_metadata = Channel.fromList(dbs.values().collect { db -> [ + db.name, workflow.sessionId, workflow.runName, workflow.manifest.name, @@ -45,20 +48,23 @@ workflow PAFTOOLS { database_version: db.version, database_path: db.root, ], - ]) + ] }) - ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata) + ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata) // [bundle name, record] - // The whole snpeff/ directory is staged so the config and the database it names - // travel together; the process overrides data.dir with -dataDir. - ch_snpeff_dir = Channel.fromPath(db.snpeff_dir, checkIfExists: true) + makeblastdb(Channel.fromList(dbs.values().collect { db -> [db.name, db.blastn] })) - makeblastdb(Channel.fromPath(db.blastn)).first().set{ch_blastdb} + // [sample_id, consensus, the sample's bundle's BLAST db] + blastn(ch_cleaned.join(ch_db) + .map { sample_id, consensus, db_name -> [db_name, sample_id, consensus] } + .combine(makeblastdb.out.db, by: 0) + .map { db_name, sample_id, consensus, blastdb -> [sample_id, consensus, blastdb] }) - blastn(ch_cleaned, ch_blastdb) - - // [sample_id, consensus, BLAST references, snpeff/] - paftools_variants(ch_cleaned.join(blastn.out.ref).combine(ch_snpeff_dir)) + // [sample_id, consensus, BLAST references, snpeff/]. The whole snpeff/ directory is + // staged so the config and the database it names travel together; the process + // overrides data.dir with -dataDir. + paftools_variants(ch_cleaned.join(blastn.out.ref).join(ch_db) + .map { sample_id, consensus, reference, db_name -> [sample_id, consensus, reference, dbs[db_name].snpeff_dir] }) paftools_variants.out.coverage .map{ it -> it[1] } @@ -92,8 +98,10 @@ workflow PAFTOOLS { // Per-sample provenance. Inner joins: prepare_consensus and blastn run errorStrategy // 'ignore', so a sample failing either leaves the chain and publishes no provenance. - ch_provenance = ch_prep_provenance - .combine(ch_pipeline_provenance) + ch_provenance = ch_prep_provenance.join(ch_db) + .map { sample_id, records, db_name -> [db_name, sample_id, records] } + .combine(ch_pipeline_provenance, by: 0) + .map { db_name, sample_id, records, pipeline -> [sample_id, records, pipeline] } .join(blastn.out.provenance) .join(paftools_variants.out.provenance) .map{ sample_id, records, pipeline, blast, paftools -> [sample_id, [pipeline] + records + [blast, paftools]] } diff --git a/workflows/short-read.nf b/workflows/short-read.nf index 5197de8..ee06447 100644 --- a/workflows/short-read.nf +++ b/workflows/short-read.nf @@ -44,6 +44,7 @@ workflow SHORT_READ { def db = resolve_db(params.db, ['blastn']) ch_workflow_metadata = Channel.value([ + db.name, workflow.sessionId, workflow.runName, workflow.manifest.name, @@ -56,7 +57,7 @@ workflow SHORT_READ { ], ]) - ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata) + ch_pipeline_provenance = pipeline_provenance(ch_workflow_metadata).map { db_name, record -> record } ch_primers = Channel.fromPath(params.primers, checkIfExists: true) @@ -66,7 +67,7 @@ workflow SHORT_READ { ch_fastq_input = Channel.fromFilePairs( params.fastq_search_path, flat: true ).map{ it -> [it[0].split('_')[0], it[1], it[2]] }.unique{ it -> it[0] } } - ch_blastn_db = Channel.fromPath(db.blastn) + ch_blastn_db = Channel.of([db.name, db.blastn]) main: @@ -92,8 +93,8 @@ workflow SHORT_READ { short_read_multiqc(short_read_fastp.out.json.mix( short_read_cutadapt.out.log, ch_fastqc_collected ).collect().ifEmpty([]) ) // Build BLAST database, then search assembled contigs against it to select references - makeblastdb(ch_blastn_db).first().set{ch_blastdb} - blastn(short_read_assembly.out.contigs, ch_blastdb) + makeblastdb(ch_blastn_db) + blastn(short_read_assembly.out.contigs.combine(makeblastdb.out.db.map { db_name, blastdb -> [blastdb] })) // NOTE: Currently mapping raw reads (ch_fastq_input). Consider using // short_read_cutadapt.out.primer_trimmed_reads instead to avoid adapter/primer artifacts. From 02938fb496e10c52208bdcc40681ee72fd43876b Mon Sep 17 00:00:00 2001 From: John Palmer <32370730+jpalmer37@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:52:03 -0400 Subject: [PATCH 3/4] Update nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index d463f28..4b3d380 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,7 @@ manifest { author = 'John Palmer' name = 'BCCDC-PHL/fluscan-nf' - version = '1.1.0' + version = '0.2.0' description = 'BCCDC-PHL FluScan' mainScript = 'main.nf' nextflowVersion = '>=22.10.0' From 0d939471d8d381b349e4ee8e0ae152d4b2b06d45 Mon Sep 17 00:00:00 2001 From: John Palmer <32370730+jpalmer37@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:53:00 -0400 Subject: [PATCH 4/4] Update nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 4b3d380..d463f28 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,7 @@ manifest { author = 'John Palmer' name = 'BCCDC-PHL/fluscan-nf' - version = '0.2.0' + version = '1.1.0' description = 'BCCDC-PHL FluScan' mainScript = 'main.nf' nextflowVersion = '>=22.10.0'