From bf8e3fcd39c2524b01d5afe9fac0a4e46fb2f506 Mon Sep 17 00:00:00 2001 From: jpalmer37 Date: Thu, 5 Nov 2026 15:39:37 -0500 Subject: [PATCH 1/2] Fix subtype parsing for strain names with spaces A reference header with no (HxNy) now fails instead of naming VCFs unknown, because fluscan-db parses that token. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- bin/build_reference_coordinate_map.py | 4 ++-- docs/database_bundles.md | 9 +++++++++ docs/watchlists.md | 4 ++-- modules/nextclade/nextclade.nf | 4 ++-- scripts/build_dataset.py | 4 ++-- tests/test_build_reference_coordinate_map.py | 4 ++-- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0fd3a02..abb9205 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -209,7 +209,7 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and canonicalise through it; a second copy lets the map be keyed under a name the merge never looks up, silently dropping a whole gene's markers. - HA marker annotation needs the Nextclade annotation to expose the HA1/HA2 - split, either as two real `CDS` features (chicken2021) or as + split, either as two real `CDS` features (hand-built datasets) or as `mature_protein_region_of_CDS` children of one HA CDS (the NCBI/RefSeq model, GsGD included). A dataset offering only a bare `HA` CDS is still unusable for HA markers and is flagged `stale_annotation`. diff --git a/bin/build_reference_coordinate_map.py b/bin/build_reference_coordinate_map.py index ce67d59..4ff99eb 100755 --- a/bin/build_reference_coordinate_map.py +++ b/bin/build_reference_coordinate_map.py @@ -305,7 +305,7 @@ def read_cds_models(path: Path, segment: str) -> dict[str, CdsModel]: amino-acid map is built at all. A mature peptide is promoted to a gene key only when no real CDS already claims that key, - so datasets exposing genuine HA1/HA2 CDS features (chicken2021) are unaffected. + so hand-built datasets exposing genuine HA1/HA2 CDS features are unaffected. """ grouped: dict[str, dict[str, Any]] = {} mature_rows: list[dict[str, Any]] = [] @@ -433,7 +433,7 @@ def detect_stale_ha_annotation( FluMutDB numbers HA by its HA1/HA2 subunits, as flumut's source reference does. The target is the flexible, user-chosen reference; it may express the split either as two - real CDS features (chicken2021) or as mature_protein_region_of_CDS children of a single + real CDS features (hand-built datasets) or as mature_protein_region_of_CDS children of a single HA CDS (the NCBI/RefSeq model, GsGD among them) -- read_cds_models accepts both, so `target_gene_keys` already carries HA1/HA2 in either case. diff --git a/docs/database_bundles.md b/docs/database_bundles.md index cf878ff..a1fda6f 100644 --- a/docs/database_bundles.md +++ b/docs/database_bundles.md @@ -116,6 +116,15 @@ the uppercase segment name, because the pipeline resolves datasets as `nextclade//`. Nothing at pipeline runtime reads `config.json`: the bundle layout is the interface, and the file is only a record of how the datasets were built. +The first word of each `reference.fasta` header names the reference in the VCF's `CHROM` +and file name. The header must also contain the subtype as `(HxNy)`, as GenBank headers do: + +``` +>PV312104.1 Influenza A virus (A/Fancy chicken/NL/FAV-0033-1-2021/2021(H5N1)) segment 4 ... +``` + +The Nextclade workflow reads the subtype from there and fails if it is missing. + **BLAST reference** — a plain nucleotide FASTA with `accession|subtype|segment` headers. The pipeline runs `makeblastdb` itself, so no index files are needed alongside it. diff --git a/docs/watchlists.md b/docs/watchlists.md index 027a3f2..e9992d0 100644 --- a/docs/watchlists.md +++ b/docs/watchlists.md @@ -88,8 +88,8 @@ HA1 numbering starts *after* HA's 16-residue signal peptide. Keying an HA1 marke | Dataset | What the VCF annotates | Correct watchlist key | |---|---|---| -| `chicken2021` | `HA1`, `HA2` as two CDS features | `HA1` / `HA2` | -| `gsgd` | `HA` CDS with `HA1`/`HA2` mature peptides | `HA1` / `HA2` | +| `chicken2021`, `gsgd` | `HA` CDS with `HA1`/`HA2` mature peptides | `HA1` / `HA2` | +| a hand-built dataset | `HA1`, `HA2` as two CDS features | `HA1` / `HA2` | | a bare `HA` CDS | `HA` | `HA` | The scanner resolves an annotation's `Feature_ID` first and its gene name second, because diff --git a/modules/nextclade/nextclade.nf b/modules/nextclade/nextclade.nf index f774edd..7ecf549 100644 --- a/modules/nextclade/nextclade.nf +++ b/modules/nextclade/nextclade.nf @@ -45,8 +45,8 @@ process nextclade_run_batch { batch.fasta REFERENCE_NAME=\$(head -n1 ${dataset_dir}/reference.fasta | sed -E 's/>(\\S+).*/\\1/g') - SUBTYPE=\$(head -n1 ${dataset_dir}/reference.fasta | sed -E 's/.+\\(\\S+\\((\\S+)\\)\\S*\\).+/\\1/g') - if echo "\$SUBTYPE" | grep -q '^>'; then SUBTYPE="unknown"; fi + SUBTYPE=\$(head -n1 ${dataset_dir}/reference.fasta | sed -nE 's/.*\\((H[0-9]+N[0-9]+)\\).*/\\1/p') + [ -n "\$SUBTYPE" ] || { echo "No (HxNy) subtype in ${dataset_dir}/reference.fasta header" >&2; exit 1; } printf -- "- process_name: nextclade_run_batch\\n" > ${segment}_nextclade_provenance.yml printf -- " tools:\\n" >> ${segment}_nextclade_provenance.yml diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py index 7313768..e014231 100644 --- a/scripts/build_dataset.py +++ b/scripts/build_dataset.py @@ -2,7 +2,7 @@ """Build Nextclade reference dataset directories from GenBank files. Reads a hand-written config.json from the dataset directory as its input manifest: -display name, ID prefix, strain, QC thresholds, and the segment list. This script is +display name, ID prefix, strain, subtype, QC thresholds, and the segment list. This script is the only consumer of that file — the pipeline resolves datasets by directory name and never reads it. @@ -234,7 +234,7 @@ def process_segment(seg_cfg, dataset_dir, cfg): seq = str(record.seq).upper() with open(outdir / "reference.fasta", "w") as fh: - fh.write(f">{seqid}\n") + fh.write(f">{seqid} Influenza A virus ({cfg['strain']}({cfg['subtype']}))\n") for chunk in textwrap.wrap(seq, 70): fh.write(chunk + "\n") diff --git a/tests/test_build_reference_coordinate_map.py b/tests/test_build_reference_coordinate_map.py index afb9329..fd9605b 100644 --- a/tests/test_build_reference_coordinate_map.py +++ b/tests/test_build_reference_coordinate_map.py @@ -4,7 +4,7 @@ 1. **HA subunits are read from either annotation shape.** FluMutDB numbers HA by its HA1/HA2 subunits. A target dataset may express that split as two real CDS features - (chicken2021) or as `mature_protein_region_of_CDS` children of one HA CDS (the + (hand-built datasets) or as `mature_protein_region_of_CDS` children of one HA CDS (the NCBI/RefSeq model, the GsGD dataset among them). The two must produce the same coordinate map -- the second one produced no HA map at all until the mature rows were read, putting all 76 of FluMutDB's HA mappings out of reach on the default dataset. @@ -50,7 +50,7 @@ def _load_module(): builder = _load_module() -# HA is modelled on chicken2021: HA1 at 77..1063 (987 nt) and HA2 at 1064..1732 (669 nt), +# HA has HA1 at 77..1063 (987 nt) and HA2 at 1064..1732 (669 nt), # adjacent, inside a 1776 nt segment. Because they abut, one CDS spanning 77..1732 with two # mature children over the same intervals is an exact re-encoding of the same protein # structure -- which is what makes the equivalence assertion below meaningful. From 51ffc9110755d8bc77e45e5c46be235e8c2f5219 Mon Sep 17 00:00:00 2001 From: jpalmer37 Date: Fri, 6 Nov 2026 13:02:29 -0500 Subject: [PATCH 2/2] Reduce subtype fix to the regex change Restores everything else in this PR to main, including the unknown fallback. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- bin/build_reference_coordinate_map.py | 4 ++-- docs/database_bundles.md | 9 --------- docs/watchlists.md | 4 ++-- modules/nextclade/nextclade.nf | 4 ++-- scripts/build_dataset.py | 4 ++-- tests/test_build_reference_coordinate_map.py | 4 ++-- 7 files changed, 11 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index abb9205..0fd3a02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -209,7 +209,7 @@ The canonical segment set is `PB2`, `PB1`, `PA`, `HA`, `NP`, `NA`, `M`, and canonicalise through it; a second copy lets the map be keyed under a name the merge never looks up, silently dropping a whole gene's markers. - HA marker annotation needs the Nextclade annotation to expose the HA1/HA2 - split, either as two real `CDS` features (hand-built datasets) or as + split, either as two real `CDS` features (chicken2021) or as `mature_protein_region_of_CDS` children of one HA CDS (the NCBI/RefSeq model, GsGD included). A dataset offering only a bare `HA` CDS is still unusable for HA markers and is flagged `stale_annotation`. diff --git a/bin/build_reference_coordinate_map.py b/bin/build_reference_coordinate_map.py index 4ff99eb..ce67d59 100755 --- a/bin/build_reference_coordinate_map.py +++ b/bin/build_reference_coordinate_map.py @@ -305,7 +305,7 @@ def read_cds_models(path: Path, segment: str) -> dict[str, CdsModel]: amino-acid map is built at all. A mature peptide is promoted to a gene key only when no real CDS already claims that key, - so hand-built datasets exposing genuine HA1/HA2 CDS features are unaffected. + so datasets exposing genuine HA1/HA2 CDS features (chicken2021) are unaffected. """ grouped: dict[str, dict[str, Any]] = {} mature_rows: list[dict[str, Any]] = [] @@ -433,7 +433,7 @@ def detect_stale_ha_annotation( FluMutDB numbers HA by its HA1/HA2 subunits, as flumut's source reference does. The target is the flexible, user-chosen reference; it may express the split either as two - real CDS features (hand-built datasets) or as mature_protein_region_of_CDS children of a single + real CDS features (chicken2021) or as mature_protein_region_of_CDS children of a single HA CDS (the NCBI/RefSeq model, GsGD among them) -- read_cds_models accepts both, so `target_gene_keys` already carries HA1/HA2 in either case. diff --git a/docs/database_bundles.md b/docs/database_bundles.md index a1fda6f..cf878ff 100644 --- a/docs/database_bundles.md +++ b/docs/database_bundles.md @@ -116,15 +116,6 @@ the uppercase segment name, because the pipeline resolves datasets as `nextclade//`. Nothing at pipeline runtime reads `config.json`: the bundle layout is the interface, and the file is only a record of how the datasets were built. -The first word of each `reference.fasta` header names the reference in the VCF's `CHROM` -and file name. The header must also contain the subtype as `(HxNy)`, as GenBank headers do: - -``` ->PV312104.1 Influenza A virus (A/Fancy chicken/NL/FAV-0033-1-2021/2021(H5N1)) segment 4 ... -``` - -The Nextclade workflow reads the subtype from there and fails if it is missing. - **BLAST reference** — a plain nucleotide FASTA with `accession|subtype|segment` headers. The pipeline runs `makeblastdb` itself, so no index files are needed alongside it. diff --git a/docs/watchlists.md b/docs/watchlists.md index e9992d0..027a3f2 100644 --- a/docs/watchlists.md +++ b/docs/watchlists.md @@ -88,8 +88,8 @@ HA1 numbering starts *after* HA's 16-residue signal peptide. Keying an HA1 marke | Dataset | What the VCF annotates | Correct watchlist key | |---|---|---| -| `chicken2021`, `gsgd` | `HA` CDS with `HA1`/`HA2` mature peptides | `HA1` / `HA2` | -| a hand-built dataset | `HA1`, `HA2` as two CDS features | `HA1` / `HA2` | +| `chicken2021` | `HA1`, `HA2` as two CDS features | `HA1` / `HA2` | +| `gsgd` | `HA` CDS with `HA1`/`HA2` mature peptides | `HA1` / `HA2` | | a bare `HA` CDS | `HA` | `HA` | The scanner resolves an annotation's `Feature_ID` first and its gene name second, because diff --git a/modules/nextclade/nextclade.nf b/modules/nextclade/nextclade.nf index 7ecf549..3fede8e 100644 --- a/modules/nextclade/nextclade.nf +++ b/modules/nextclade/nextclade.nf @@ -45,8 +45,8 @@ process nextclade_run_batch { batch.fasta REFERENCE_NAME=\$(head -n1 ${dataset_dir}/reference.fasta | sed -E 's/>(\\S+).*/\\1/g') - SUBTYPE=\$(head -n1 ${dataset_dir}/reference.fasta | sed -nE 's/.*\\((H[0-9]+N[0-9]+)\\).*/\\1/p') - [ -n "\$SUBTYPE" ] || { echo "No (HxNy) subtype in ${dataset_dir}/reference.fasta header" >&2; exit 1; } + SUBTYPE=\$(head -n1 ${dataset_dir}/reference.fasta | sed -E 's/.+\\([^()]+\\((\\S+)\\)\\S*\\).+/\\1/g') + if echo "\$SUBTYPE" | grep -q '^>'; then SUBTYPE="unknown"; fi printf -- "- process_name: nextclade_run_batch\\n" > ${segment}_nextclade_provenance.yml printf -- " tools:\\n" >> ${segment}_nextclade_provenance.yml diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py index e014231..7313768 100644 --- a/scripts/build_dataset.py +++ b/scripts/build_dataset.py @@ -2,7 +2,7 @@ """Build Nextclade reference dataset directories from GenBank files. Reads a hand-written config.json from the dataset directory as its input manifest: -display name, ID prefix, strain, subtype, QC thresholds, and the segment list. This script is +display name, ID prefix, strain, QC thresholds, and the segment list. This script is the only consumer of that file — the pipeline resolves datasets by directory name and never reads it. @@ -234,7 +234,7 @@ def process_segment(seg_cfg, dataset_dir, cfg): seq = str(record.seq).upper() with open(outdir / "reference.fasta", "w") as fh: - fh.write(f">{seqid} Influenza A virus ({cfg['strain']}({cfg['subtype']}))\n") + fh.write(f">{seqid}\n") for chunk in textwrap.wrap(seq, 70): fh.write(chunk + "\n") diff --git a/tests/test_build_reference_coordinate_map.py b/tests/test_build_reference_coordinate_map.py index fd9605b..afb9329 100644 --- a/tests/test_build_reference_coordinate_map.py +++ b/tests/test_build_reference_coordinate_map.py @@ -4,7 +4,7 @@ 1. **HA subunits are read from either annotation shape.** FluMutDB numbers HA by its HA1/HA2 subunits. A target dataset may express that split as two real CDS features - (hand-built datasets) or as `mature_protein_region_of_CDS` children of one HA CDS (the + (chicken2021) or as `mature_protein_region_of_CDS` children of one HA CDS (the NCBI/RefSeq model, the GsGD dataset among them). The two must produce the same coordinate map -- the second one produced no HA map at all until the mature rows were read, putting all 76 of FluMutDB's HA mappings out of reach on the default dataset. @@ -50,7 +50,7 @@ def _load_module(): builder = _load_module() -# HA has HA1 at 77..1063 (987 nt) and HA2 at 1064..1732 (669 nt), +# HA is modelled on chicken2021: HA1 at 77..1063 (987 nt) and HA2 at 1064..1732 (669 nt), # adjacent, inside a 1776 nt segment. Because they abut, one CDS spanning 77..1732 with two # mature children over the same intervals is an exact re-encoding of the same protein # structure -- which is what makes the equivalence assertion below meaningful.