diff --git a/Dockerfile b/Dockerfile index 422e2bf83..bc5a52c45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 - COPY poetry.lock pyproject.toml ./ # installs runtime dependencies to $VIRTUAL_ENV -RUN poetry install --no-root --extras server +RUN poetry install --no-root --extras server --no-directory COPY alembic /code/alembic COPY alembic.ini /code/alembic.ini COPY src /code/src diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..16f8319bc --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +VENV := .venv/bin + +.DEFAULT_GOAL := help + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}' + +.PHONY: dev +dev: ## Install deps including local editable variant-annotation + poetry install --extras server + +.PHONY: test +test: ## Run the test suite + $(VENV)/pytest tests/ + +.PHONY: lint +lint: ## Check code with ruff + $(VENV)/ruff check src/ tests/ + +.PHONY: format +format: ## Format code with ruff + $(VENV)/ruff format src/ tests/ diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py new file mode 100644 index 000000000..32c752a6b --- /dev/null +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -0,0 +1,1326 @@ +"""Backfill the Allele substrate (``mapping_records`` / ``alleles`` / +``mapping_record_alleles`` + the annotation link tables) from existing +``mapped_variants`` rows. + +Context +------- +The annotation/serving layer is migrating off ``MappedVariant`` onto a +parallel-tables model:: + + Variant ─< MappingRecord (1 live/variant, ValidTime) ─< MappingRecordAllele (ValidTime) >─ Allele (dedup by vrs_digest) + +This script populates the new tables from the *existing* ``mapped_variants`` data so +the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV +rewrites and the serving readers) resolves for historical score sets, which are +otherwise empty in the new substrate. + +This is the **reshape half of the hybrid migration**: it reconstructs the +authoritative spine deterministically from old data with no external-service +calls. It deliberately does **not** reproduce the reverse-translation +equivalence fan-out (the per-level coding/genomic/protein alleles and their +``projection_group`` pairing) — that data was never computed for old mappings +(``MappedVariant`` holds a single ``post_mapped`` VRS plus cross-level HGVS +*strings*, and an Allele needs a real ``vrs_digest`` that a string cannot +supply). Restoring that richness is the decoupled *enrichment* half: queue the +reverse-translation + annotation-refresh jobs per score set on the worker after +this backfill lands. Serving is correct at the measured level as soon as this +runs; enrichment only adds cross-level breadth and blocks nothing. + +What is reconstructed +--------------------- +* **MappingRecord history** — every ``MappedVariant`` version of a variant with a resolvable sequence + level becomes a ``MappingRecord`` with a gap-free ``[valid_from, valid_to)`` window derived from the + versions' ``mapped_date`` ordering. The ``current=True`` version is the live tail (``valid_to IS + NULL``). History dates are inherently approximate: ``MappedVariant`` records versioning with a + ``current`` boolean, not transaction time, so a window boundary is the successor's ``mapped_date``, + not the exact instant the swap happened. A version whose sequence level can't be attributed is + skipped **individually** (its siblings still get their records — see Idempotency below); if that + version happens to be the live one, the variant has no live record and stays eligible for retry. + Live dates that precede a historical version's are a logged anomaly (a likely re-map candidate), not + a fatal condition — see :func:`_reconstruct_windows`. +* **Authoritative Allele** — from each version's ``post_mapped`` VRS, deduplicated by ``vrs_digest`` + against an in-memory, per-chunk cache (:func:`_authoritative_allele` — not the live job's + ``get_or_create_allele``, which queries per call; see :func:`do_migration`'s docstring for why). + Carries the CAID and the assay-level HGVS + across so the allele-keyed serving endpoints resolve. The assay-level HGVS itself is read through a + three-tier fallback (:func:`_hgvs_assay_level_with_source`): ``mv.hgvs_assay_level`` directly, else + the per-level column matching the resolved level (``hgvs_g``/``hgvs_c``/``hgvs_p`` — backfilled later + and never caught up on a large share of old data), else the ``post_mapped`` VRS blob's own + ``expressions`` (the mapper sometimes stamps or reconstructs an HGVS expression directly on the VRS + Allele even when neither dedicated column got populated). Even with all three tiers, some legacy rows + carry no recoverable HGVS at all — that gap is real and needs the mapper re-run to close, not another + migration-time trick. Reading only ``hgvs_assay_level`` (as this script originally did) left most + reconstructed Alleles (and ``MappingRecord.hgvs_assay_level``, and therefore its derived + ``transcript``) NULL even when a real string was recoverable — silently starving any downstream pass + that resolves a transcript from it. +* **MappingRecordAllele** — one ``is_authoritative=True`` link per record on the + record's validity window. ``projection_group`` stays NULL (no fan-out). +* **Annotation timelines** — reconstructed across *every* mapping version, not just the live one: + ``gnomad_variants`` → ``gnomad_allele_links``, ``clinical_controls`` → ``clinvar_allele_links``, + VEP columns → ``vep_allele_consequences``. Each mapping version's bundled annotations are captured + against that version's window and, per allele, merged per source key (gnomAD variant / ClinVar + control / consequence value) into maximal intervals — so a continuously-held annotation is one link + dated from its **earliest** mapping version, open-ended only if it reaches the live record. A + release change that coincided with a re-map surfaces as a superseding closed+live pair. This is + faithful (the old jobs wrote annotations on the current mapping row, so their changes aligned to + re-map boundaries), just **coarse** — sub-mapping-version churn collapses to the latest value in + that window, and boundary timestamps are day-grained. The valid-time axis is uniformly the mapping + date across sources (VEP's real ``access_date`` is kept in its own audit column, not as + ``valid_from``, so ``as_of`` stays coherent). VEP has no Ensembl release string in old data, but + ``access_date`` is enough to derive one: ``source_version`` is looked up from a snapshotted table of + Ensembl release dates (:data:`_ENSEMBL_VEP_RELEASE_DATES`), not stamped as an opaque sentinel — see + :func:`_resolve_vep_source_version`. Only a missing ``access_date``, or one older than the table's + earliest known release, falls back to the ``"legacy"`` sentinel. + + **Exception:** a gnomAD or ClinVar window that would otherwise land live on a **protein-level** allele + is force-closed at the migration's own run time instead (VEP is unaffected — see below). The old + model could proxy both onto protein-level rows because there was nowhere else to attach them; the new + model resolves them correctly through the CA/PA link graph onto nucleotide siblings instead + (``lib/allele_measurements.py``), so carrying the proxy forward live would just create a second, + competing provenance story for the same value. Applied uniformly even though gnomAD is not known to + have actually been proxied this way in practice (ClinVar was) — it is the same underlying anti-pattern + (a nucleotide-level concept attached to a protein-level allele), so the guard costs nothing to keep + general. The historical fact is kept (closed, not deleted) for point-in-time audit; live annotations + for these alleles return once cross-level annotation fan-out re-derives them properly. + +Idempotency / no double-write +----------------------------- +A variant is either old-substrate or new (the parallel-tables invariant); native new-model variants +have no ``mapped_variants``. The migration therefore processes only variants that have +``mapped_variants`` and **no live** ``MappingRecord`` yet — live, not "any", because a variant whose +live mapped_variant version has no resolvable sequence level is left with only historical records (see +above) and must stay selectable so a later re-map or QC fix can complete it. Within a selected variant, +each window is additionally guarded by a ``(variant_id, mapped_date)`` existence check, so a re-run +that revisits a partially-migrated variant only fills in the windows still missing rather than +duplicating the ones a prior pass already wrote. Annotation links are existence-checked before +insertion, so a partial run resumes cleanly; run ``rebuild-annotations`` after resuming a +partially-migrated variant to pick up any timelines the skipped windows didn't get to build. + +Standalone convention +--------------------- +Follows the ``migrate_jsonb_ranges_to_table_rows.py`` pattern: invoked outside +``alembic upgrade`` (it can take a long time on production data), commits in +variant batches, and offers ``verify`` / ``rollback`` commands. + +Usage +----- +:: + + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate # run + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate verify # inspect without writing + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rollback # remove backfilled records (destructive) + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rebuild-annotations # (re)build annotation links for already-migrated variants + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --score-set urn:mavedb:00000001-a-1 + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --batch-size 1000 --dry-run +""" + +from __future__ import annotations + +import argparse +import logging +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date, datetime, time, timezone +from time import perf_counter # `time` above is datetime.time; import the timer directly +from typing import Optional + +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, configure_mappers + +from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import — register all mappers for configure_mappers() +from mavedb.db.session import SessionLocal +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +configure_mappers() + +logger = logging.getLogger(__name__) + +# Old VEP data carries no Ensembl release string, only an access_date — but source_version is derivable: +# Ensembl releases (and VEP's version with them) are coordinated, dated, and public +# (https://github.com/Ensembl/ensembl-vep/releases), so the release active on a given access_date can +# be looked up rather than stamped as an opaque sentinel. Each entry is a release's *earliest* GitHub +# tag date (patch tags like 92.1-92.6 are hotfixes within release 92, not new majors); an access_date +# is attributed to the newest release whose date is <= it. Snapshotted from the GitHub API on +# 2026-07-20 (covers releases 92 through 116); this table does not self-update, so re-run the same +# query and extend it before backfilling data annotated under a release newer than the last entry here. +_ENSEMBL_VEP_RELEASE_DATES: list[tuple[date, str]] = [ + (date(2018, 4, 12), "92"), + (date(2018, 7, 17), "93"), + (date(2018, 10, 2), "94"), + (date(2019, 1, 10), "95"), + (date(2019, 4, 8), "96"), + (date(2019, 7, 3), "97"), + (date(2019, 9, 27), "98"), + (date(2020, 1, 16), "99"), + (date(2020, 4, 29), "100"), + (date(2020, 8, 20), "101"), + (date(2020, 11, 30), "102"), + (date(2021, 2, 16), "103"), + (date(2021, 5, 5), "104"), + (date(2021, 12, 9), "105"), + (date(2022, 4, 14), "106"), + (date(2022, 7, 13), "107"), + (date(2022, 10, 20), "108"), + (date(2023, 2, 8), "109"), + (date(2023, 7, 18), "110"), + (date(2024, 1, 11), "111"), + (date(2024, 5, 16), "112"), + (date(2024, 10, 18), "113"), + (date(2025, 5, 8), "114"), + (date(2025, 9, 3), "115"), + (date(2026, 6, 10), "116"), +] + +# Fallback sentinel for the cases the release-date table can't resolve: no access_date at all, or an +# access_date older than the table's earliest known release (GitHub has no tags before release/92, so +# anything before 2018-04-12 is out of range). VepAlleleConsequence.source_version is NOT NULL, and a +# refresh run should treat this value as unversioned and re-confirm it against the current release +# rather than trusting it as current — the same behavior the sentinel always had. +LEGACY_VEP_SOURCE_VERSION = "legacy" + + +def _resolve_vep_source_version(access_date: Optional[date]) -> str: + """Map a legacy VEP ``access_date`` to the Ensembl release active on that day. + + Returns :data:`LEGACY_VEP_SOURCE_VERSION` when ``access_date`` is missing or predates the earliest + release the table knows about — both cases where the release genuinely can't be determined, as + opposed to just not being in the table's covered range for lack of maintenance. + """ + if access_date is None or access_date < _ENSEMBL_VEP_RELEASE_DATES[0][0]: + return LEGACY_VEP_SOURCE_VERSION + resolved = _ENSEMBL_VEP_RELEASE_DATES[0][1] + for release_date, version in _ENSEMBL_VEP_RELEASE_DATES: + if release_date > access_date: + break + resolved = version + return resolved + + +def _as_dt(d: date) -> datetime: + """Lift a ``date`` to a timezone-aware ``datetime`` for a ValidTime boundary. + + ValidTime windows are ``DateTime(timezone=True)``; ``MappedVariant`` versioning + is day-grained (``mapped_date``), so midnight UTC is the reconstructed boundary. + """ + return datetime.combine(d, time.min, tzinfo=timezone.utc) + + +def _resolve_level(mv: MappedVariant) -> Optional[SequenceLevel]: + """The sequence level a ``MappedVariant`` was assayed/aligned at. + + ``alignment_level`` is populated on every row by the target-gene-mapping QC + backfill and is the direct source (the mapping job sets a record's + ``assay_level`` and ``alignment_level`` to the same value). For the rare + legacy row the QC backfill could not attribute, fall back to whichever mapped + HGVS column is populated. Returns ``None`` when neither signal is present — + the record cannot be created (``assay_level`` is NOT NULL) and the caller + skips the variant with a logged anomaly. + """ + if mv.alignment_level is not None: + return mv.alignment_level + if mv.hgvs_g: + return SequenceLevel.genomic + if mv.hgvs_c: + return SequenceLevel.cdna + if mv.hgvs_p: + return SequenceLevel.protein + return None + + +_LEVEL_HGVS_COLUMN = { + SequenceLevel.genomic: "hgvs_g", + SequenceLevel.cdna: "hgvs_c", + SequenceLevel.protein: "hgvs_p", +} + + +def _hgvs_assay_level_with_source(mv: MappedVariant, level: SequenceLevel) -> tuple[Optional[str], str]: + """Resolve the assay-level HGVS string, reporting which of three tiers supplied it. + + ``mv.hgvs_assay_level`` is frequently ``NULL`` on legacy rows — it was backfilled later and never + caught up on a large share of old data — and reading only it (as this script originally did) meant + most reconstructed Alleles got a NULL ``hgvs_g``/``hgvs_c``/``hgvs_p`` and + ``MappingRecord.hgvs_assay_level`` even when the real string was recoverable. Since + ``MappingRecord.transcript`` derives from ``hgvs_assay_level``, that silently starved any downstream + pass that resolves a transcript from it (e.g. cross-level annotation fan-out). + + Three tiers, in order: + + 1. ``"direct"`` — ``mv.hgvs_assay_level`` itself. + 2. ``"level_column"`` — the per-level column matching ``level`` (``hgvs_g``/``hgvs_c``/``hgvs_p``), + mirroring :func:`_resolve_level`'s own fallback (same column), just for the *value* rather than + the level. + 3. ``"vrs_expression"`` — the ``post_mapped`` VRS blob's own ``expressions``. Some legacy rows have + *neither* dedicated column populated, but the mapper stamped an HGVS expression directly onto + the VRS Allele it produced (``dcd_mapping`` sets this unconditionally for coding-layer alleles, + and reconstructs it for genomic/protein when accession resolution succeeds — so it is not + universal either, just an independent, additional chance). Reuses + :func:`mavedb.lib.variants.get_hgvs_from_post_mapped`, the same extraction serving already relies + on, rather than re-parsing VRS JSON here. A malformed or VRS-1.x blob raises there; caught here + and treated as "no expression" so one bad blob doesn't abort the whole variant. + + Returns ``(None, "none")`` when no tier resolves anything — some legacy data genuinely carries no + recoverable HGVS at all, and that gap cannot be closed without re-running the mapper. + """ + if mv.hgvs_assay_level: + return mv.hgvs_assay_level, "direct" + + level_value = getattr(mv, _LEVEL_HGVS_COLUMN[level]) + if level_value: + return level_value, "level_column" + + try: + vrs_value = get_hgvs_from_post_mapped(mv.post_mapped) + except (KeyError, ValueError): + vrs_value = None + if vrs_value: + return vrs_value, "vrs_expression" + + return None, "none" + + +def _resolve_hgvs_assay_level(mv: MappedVariant, level: SequenceLevel) -> Optional[str]: + """The assay-level HGVS string for a version — see :func:`_hgvs_assay_level_with_source`.""" + return _hgvs_assay_level_with_source(mv, level)[0] + + +@dataclass +class _WindowResult: + """``_reconstruct_windows``'s output: the ordered windows plus an anomaly flag. + + Kept as one return value (rather than ``None``-means-skip on windows, anomaly bundled in) so a + caller can't forget to check the flag the way a bare tuple invites. + """ + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] + live_precedes_history: bool + + +def _reconstruct_windows(mvs: list[MappedVariant]) -> Optional[_WindowResult]: + """Order a variant's ``MappedVariant`` versions and assign ValidTime windows. + + Returns ``(mv, valid_from, valid_to)`` per version, gap-free: each version's + ``valid_to`` is its successor's ``valid_from``, and the live tail's is + ``None``. The ``current=True`` version is the live tail regardless of its + ``mapped_date``. Returns ``None`` (skip) when no version is current — a + variant with mappings but no live one is a source anomaly we do not guess a + live state for. + + ``live_precedes_history=True`` flags the case where the live version's ``mapped_date`` is earlier + than some historical version's — a data-quality anomaly (``current`` should always be the newest). + When it happens, the pairwise clamp below still prevents inverted windows, but at the cost of + collapsing the affected historical window to zero-width (invisible to ``as_of``) rather than + losing data outright: the caller surfaces the flag as a loud, variant-attributed warning instead + of silently reordering, since it likely means the source variant wants a re-map. + """ + current = [m for m in mvs if m.current] + history = [m for m in mvs if not m.current] + + if not current: + return None + if len(current) == 1: + live = current[0] + else: + # Multiple live rows is a source anomaly; keep the newest live and demote + # the rest into history so no data is dropped. + current.sort(key=lambda m: (m.mapped_date, m.id)) + live = current[-1] + history.extend(current[:-1]) + logger.warning("Variant %s has %d current mapped_variants; keeping newest live.", live.variant_id, len(current)) + + history.sort(key=lambda m: (m.mapped_date, m.id)) + ordered = history + [live] + live_precedes_history = bool(history) and live.mapped_date < history[-1].mapped_date + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] = [] + for i, m in enumerate(ordered): + valid_from = _as_dt(m.mapped_date) + valid_to: Optional[datetime] = None + if i + 1 < len(ordered): + successor = _as_dt(ordered[i + 1].mapped_date) + # Same-day or out-of-order successor: clamp so the window is never + # inverted (a zero-width [t, t) window is simply invisible to as_of). + valid_to = max(successor, valid_from) + windows.append((m, valid_from, valid_to)) + return _WindowResult(windows=windows, live_precedes_history=live_precedes_history) + + +def _fill_missing_allele_fields(allele: Allele, mv: MappedVariant, level: SequenceLevel) -> None: + """Backfill identity fields onto a (possibly pre-existing, deduped) allele. + + Dedup means the first variant to mint an allele sets its fields; a later + variant sharing the ``vrs_digest`` may carry the CAID or level HGVS the first + lacked. Those are content-derived (deterministic from the allele), so + fill-if-missing is safe and never overwrites a populated value. + """ + if allele.clingen_allele_id is None and mv.clingen_allele_id: + allele.clingen_allele_id = mv.clingen_allele_id + + hgvs_col = _LEVEL_HGVS_COLUMN[level] + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if getattr(allele, hgvs_col) is None and hgvs_value: + setattr(allele, hgvs_col, hgvs_value) + + +def _authoritative_allele( + db: Session, mv: MappedVariant, level: SequenceLevel, allele_cache: dict[str, Allele] +) -> Optional[Allele]: + """Get-or-create the authoritative allele from ``mv.post_mapped``, via an in-memory dedup cache. + + Returns ``None`` for versions with no post-mapped representation (failed or benign-absent variants + get a record but no linked allele, matching the mapping job). + + Unlike :func:`mavedb.lib.variant_translations.get_or_create_allele` (used by the live mapping job), + this does **not** query the database or flush per call. ``allele_cache`` is pre-seeded by the + caller with one bulk ``vrs_digest IN (...)`` query per chunk (see :func:`do_migration`), and this + function keeps it updated in-memory as new alleles are drafted — so a digest shared by two windows + in the *same* chunk (a common dedup case) resolves from the dict, not a second round trip. This is + safe only because the session runs with ``autoflush=False`` (a query wouldn't see an unflushed + insert anyway) and because the caller flushes once per variant, not per window: an actual + ``uq_alleles_vrs_digest`` collision (e.g. a concurrent live mapping run) still surfaces at that + flush, inside the same per-variant SAVEPOINT it always has — this only removes the redundant reads, + not the write-time integrity check. + """ + post_mapped = mv.post_mapped or {} + digest = post_mapped.get("id") + if not digest: + return None + + existing = allele_cache.get(digest) + if existing is not None: + _fill_missing_allele_fields(existing, mv, level) + return existing + + hgvs_value = _resolve_hgvs_assay_level(mv, level) + allele = Allele( + vrs_digest=digest, + level=level, + hgvs_g=hgvs_value if level == SequenceLevel.genomic else None, + hgvs_c=hgvs_value if level == SequenceLevel.cdna else None, + hgvs_p=hgvs_value if level == SequenceLevel.protein else None, + clingen_allele_id=mv.clingen_allele_id, + post_mapped=post_mapped, + ) + db.add(allele) + allele_cache[digest] = allele + return allele + + +@dataclass +class _AnnotationObservation: + """One mapping version's annotation payload for an authoritative allele, tagged with that + version's validity window. + + Built fresh from the DB per allele-batch (:func:`_observations_for_alleles`) rather than collected + inline during record creation — so a deduped allele shared by several variants, or spanning a + variant's re-map history, still yields one merged timeline rather than per-version fragments (see + :func:`_write_annotation_timelines`), without needing every observation for the whole run held in + memory at once. + """ + + valid_from: datetime + valid_to: Optional[datetime] + gnomad_variant_ids: tuple[int, ...] + clinvar_control_ids: tuple[int, ...] + vep_consequence: Optional[str] + vep_access_date: Optional[date] + + +def _observation_for( + mv: MappedVariant, valid_from: datetime, valid_to: Optional[datetime] +) -> Optional[_AnnotationObservation]: + """Capture a mapping version's annotations for its window, or ``None`` if it carries none.""" + gnomad_ids = tuple(g.id for g in mv.gnomad_variants) + clinvar_ids = tuple(c.id for c in mv.clinical_controls) + if not gnomad_ids and not clinvar_ids and not mv.vep_functional_consequence: + return None + return _AnnotationObservation( + valid_from=valid_from, + valid_to=valid_to, + gnomad_variant_ids=gnomad_ids, + clinvar_control_ids=clinvar_ids, + vep_consequence=mv.vep_functional_consequence, + vep_access_date=mv.vep_access_date, + ) + + +def _merge_intervals( + windows: list[tuple[datetime, Optional[datetime]]], +) -> list[tuple[datetime, Optional[datetime]]]: + """Merge overlapping/adjacent ``[valid_from, valid_to)`` windows; ``valid_to=None`` is open-ended. + + An annotation held across a run of mapping versions produces contiguous windows (and overlapping + ones when a deduped allele is shared by several variants). Merging collapses each continuous run + into one link — ``valid_from`` = its earliest mapping date, ``valid_to`` open only if the run + reaches the live record — and closes a link only where the annotation actually lapses, rather than + fabricating a boundary at every re-map. + """ + ordered = sorted(windows, key=lambda w: w[0]) + merged: list[tuple[datetime, Optional[datetime]]] = [] + cur_from, cur_to = ordered[0] + for vf, vt in ordered[1:]: + if cur_to is None or vf <= cur_to: # overlap/adjacency; an open-ended window absorbs the rest + cur_to = None if (cur_to is None or vt is None) else max(cur_to, vt) + else: + merged.append((cur_from, cur_to)) + cur_from, cur_to = vf, vt + merged.append((cur_from, cur_to)) + return merged + + +def _allele_has_live_link(db: Session, model: type, allele_id: int) -> bool: + """Whether ``allele_id`` already has a live (``valid_to IS NULL``) link in ``model``. + + Sees **committed** rows only — the session runs ``autoflush=False``, so this does not observe + not-yet-flushed adds from the current pass. It therefore catches prior-run and concurrently- + committed live links (e.g. from the mapping pipeline running alongside this backfill); single-live + within one pass is enforced separately, in memory, by :func:`_write_allele_timeline`. + """ + return db.scalar(sa.select(model.id).where(model.allele_id == allele_id, model.valid_to.is_(None))) is not None + + +def _emit_gnomad_link( + db: Session, + allele_id: int, + gnomad_variant_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(GnomadAlleleLink.id).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.gnomad_variant_id == gnomad_variant_id, + GnomadAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + db.add( + GnomadAlleleLink( + allele_id=allele_id, gnomad_variant_id=gnomad_variant_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["gnomad_links"] += 1 + + +def _emit_clinvar_link( + db: Session, + allele_id: int, + clinvar_control_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(ClinvarAlleleLink.id).where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.clinvar_control_id == clinvar_control_id, + ClinvarAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run; ClinVar is multi-live per (allele, control) + return + db.add( + ClinvarAlleleLink( + allele_id=allele_id, clinvar_control_id=clinvar_control_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["clinvar_links"] += 1 + + +def _emit_vep_consequence( + db: Session, + allele_id: int, + consequence: str, + access_date: date, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(VepAlleleConsequence.id).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.functional_consequence == consequence, + VepAlleleConsequence.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + source_version = _resolve_vep_source_version(access_date) + stats["vep_source_version_legacy_fallback" if source_version == LEGACY_VEP_SOURCE_VERSION else "vep_source_version_resolved"] += 1 + db.add( + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["vep_consequences"] += 1 + + +def _write_allele_timeline( + db: Session, allele_id: int, obs_list: list[_AnnotationObservation], stats: Counter, run_at: datetime +) -> None: + """Build and emit one allele's merged annotation link timelines (no flush/commit here). + + Keyed per source — gnomAD by ``gnomad_variant_id``, ClinVar by ``clinvar_control_id``, VEP by + consequence value — each key's windows are merged so a continuously-held annotation is a single + link dated from its earliest mapping version, open-ended only if it reaches the live record. + + **Single-live-per-allele** for gnomAD/VEP is enforced with an in-memory flag seeded from committed + state (:func:`_allele_has_live_link`). This is the fix for the ``autoflush=False`` session: a + deduped allele linked to two *different* gnomAD variants (from two variants sharing the allele) + would otherwise emit two live links and violate ``uq_gnomad_allele_links_live`` at flush, since the + per-add DB check can't see the earlier unflushed add. The flag also folds in a link already live + from a prior run or the concurrent mapping pipeline. ClinVar is multi-live per ``(allele, control)``. + + **Protein-level gnomAD/ClinVar are a legacy proxy the new model doesn't carry forward live.** The + old model could attach ClinVar significance and gnomAD frequency to protein-level ``MappedVariant`` + rows as a proxy — there was nowhere else to put them. Both are intrinsically nucleotide-level + concepts (a ClinVar submission and a gnomAD allele frequency are both keyed to a genomic variant, + not a protein change), and the new model resolves them correctly for a protein allele via the CA/PA + link graph onto its nucleotide siblings (see ``lib/allele_measurements.py``), not by direct + attachment. A live direct link here would just be a second, competing provenance story for the same + value. So a protein allele's gnomAD/ClinVar window is force-closed at ``run_at`` (the migration's + own run time) instead of staying open-ended: the historical fact that this had been proxy-annotated + is kept for point-in-time audit, but it stops being *live* data at the moment of reshape, to be + replaced once cross-level annotation fan-out re-derives it correctly through the link graph. VEP is + exempt — a functional consequence is computed relative to the assayed representation itself, so it + is not a proxied nucleotide-level value the way gnomAD/ClinVar are. + """ + gnomad_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + clinvar_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_windows: dict[str, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_access: dict[str, list[date]] = defaultdict(list) + + for obs in obs_list: + for gnomad_variant_id in obs.gnomad_variant_ids: + gnomad_windows[gnomad_variant_id].append((obs.valid_from, obs.valid_to)) + for clinvar_control_id in obs.clinvar_control_ids: + clinvar_windows[clinvar_control_id].append((obs.valid_from, obs.valid_to)) + if obs.vep_consequence: + vep_windows[obs.vep_consequence].append((obs.valid_from, obs.valid_to)) + if obs.vep_access_date is not None: + vep_access[obs.vep_consequence].append(obs.vep_access_date) + + is_protein = False + if gnomad_windows or clinvar_windows: + allele_level = db.scalar(sa.select(Allele.level).where(Allele.id == allele_id)) + is_protein = allele_level == SequenceLevel.protein.value + + gnomad_live_taken = _allele_has_live_link(db, GnomadAlleleLink, allele_id) + for gnomad_variant_id, windows in gnomad_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + # Force-closed, not left live: not subject to the single-live guard below, since it + # never becomes a live row that could collide with uq_gnomad_allele_links_live. + valid_to = run_at + stats["protein_gnomad_links_closed"] += 1 + elif valid_to is None: + if gnomad_live_taken: + logger.warning( + "Allele %s already has a live gnomAD link; skipping gnomad_variant %s.", + allele_id, + gnomad_variant_id, + ) + continue + gnomad_live_taken = True + _emit_gnomad_link(db, allele_id, gnomad_variant_id, valid_from, valid_to, stats) + + for clinvar_control_id, windows in clinvar_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + valid_to = run_at + stats["protein_clinvar_links_closed"] += 1 + _emit_clinvar_link(db, allele_id, clinvar_control_id, valid_from, valid_to, stats) + + vep_live_taken = _allele_has_live_link(db, VepAlleleConsequence, allele_id) + for consequence, windows in vep_windows.items(): + observed_dates = vep_access[consequence] + for valid_from, valid_to in _merge_intervals(windows): + if valid_to is None: + if vep_live_taken: + logger.warning("Allele %s already has a live VEP consequence; skipping %r.", allele_id, consequence) + continue + vep_live_taken = True + access_date = min(observed_dates) if observed_dates else valid_from.date() + _emit_vep_consequence(db, allele_id, consequence, access_date, valid_from, valid_to, stats) + + +def _write_annotation_timelines( + db: Session, observations: dict[int, list[_AnnotationObservation]], stats: Counter, run_at: datetime +) -> None: + """Write each allele's annotation timelines, isolating conflicts per allele. + + Each allele is built + flushed inside its own SAVEPOINT: if a live-uniqueness collision still + slips through the in-memory guard — e.g. the concurrently-running mapping pipeline commits a live + link for a shared allele between the guard's committed-state read and this flush — only that + allele's links roll back (counted as ``annotation_conflicts``), and the rest of the pass proceeds + instead of the whole final commit aborting. + + ``run_at`` is a single timestamp for the whole pass (not re-read per allele), so every protein-level + ClinVar link force-closed by :func:`_write_allele_timeline` in this run shares the same cutover + instant rather than drifting with wall-clock time across a long batch. + """ + for allele_id, obs_list in observations.items(): + try: + with db.begin_nested(): + _write_allele_timeline(db, allele_id, obs_list, stats, run_at) + db.flush() # surface any live-uniqueness violation inside this allele's savepoint + except IntegrityError: + stats["annotation_conflicts"] += 1 + logger.warning("Annotation links for allele %s conflicted (live elsewhere); skipped.", allele_id) + + +def _observations_for_alleles(db: Session, allele_ids: list[int]) -> dict[int, list[_AnnotationObservation]]: + """Rebuild annotation observations for one batch of alleles, from fresh DB state. + + Scoped by allele id, not by variant/chunk: the merge :func:`_write_allele_timeline` performs is only + ever correct once *every* mapping version that ever authoritatively linked to a given allele has been + observed, and alleles are shared arbitrarily across variants — a variant-id chunk boundary says + nothing about which alleles are "done" (an allele's contributing variants can land in different + chunks). Batching by allele id instead means every batch's observations are complete for the alleles + in it, however many separate variant-chunks originally fed those alleles. + + Pairs each authoritative ``MappingRecord`` to its source ``MappedVariant`` version by + ``(variant_id, mapped_date)`` — the record was created with that version's ``mapped_date`` — mirroring + the pairing :func:`_migrate_variant` used when the record was first built. + """ + links = db.execute( + sa.select( + MappingRecordAllele.allele_id, + MappingRecord.variant_id, + MappingRecord.mapped_date, + MappingRecord.valid_from, + MappingRecord.valid_to, + ) + .join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .where(MappingRecordAllele.allele_id.in_(allele_ids), MappingRecordAllele.is_authoritative) + ).all() + if not links: + return {} + + variant_ids = {variant_id for _, variant_id, _, _, _ in links} + mv_by_key: dict[tuple[int, date], MappedVariant] = { + (mv.variant_id, mv.mapped_date): mv + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(variant_ids))).all() + } + + observations: dict[int, list[_AnnotationObservation]] = defaultdict(list) + for allele_id, variant_id, mapped_date, valid_from, valid_to in links: + mv = mv_by_key.get((variant_id, mapped_date)) + if mv is None: + continue # authoritative link survives independently of its source mapped_variant row + observation = _observation_for(mv, valid_from, valid_to) + if observation is not None: + observations[allele_id].append(observation) + return observations + + +def _backfill_annotation_timelines( + db: Session, allele_ids: list[int], *, batch_size: int, run_at: datetime, stats: Counter +) -> None: + """Build and commit annotation link timelines for ``allele_ids``, batch by batch. + + Each batch re-reads its own observations fresh from the DB (:func:`_observations_for_alleles`) and + commits before moving to the next, so memory is bounded by ``batch_size`` alleles at a time + regardless of how many millions of variants fed into ``allele_ids``. This replaces the previous + design, which accumulated every ``_AnnotationObservation`` for the *whole run* in memory before + writing anything at the very end — correct, but the accumulator's unbounded growth over a + multi-million-variant run put steadily increasing pressure on the GC, degrading throughput well + before the run completed (not just a flat per-call cost, but one that compounded with how much had + already been processed). + """ + total = len(allele_ids) + pass_start = perf_counter() + for offset in range(0, total, batch_size): + chunk = allele_ids[offset : offset + batch_size] + observations = _observations_for_alleles(db, chunk) + if observations: + _write_annotation_timelines(db, observations, stats, run_at) + db.commit() + processed = min(offset + batch_size, total) + elapsed = perf_counter() - pass_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Built timelines for {processed}/{total} allele(s)... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + +def _migrate_variant( + db: Session, + variant_id: int, + mvs: list[MappedVariant], + stats: Counter, + skipped: dict[str, list[int]], + annotation_manager: AnnotationStatusManager, + existing_records: set[tuple[int, date]], + allele_cache: dict[str, Allele], +) -> set[int]: + """Reconstruct as much of a variant's record history as its data supports, returning the ids of + every allele this variant authoritatively touched. + + Also records one ``VRS_MAPPING`` event per version created (variant-subject) and one + ``CLINGEN_ALLELE_ID`` / ``MAPPED_HGVS`` event per allele-touch (allele-subject), all reason + :data:`EventReason.MIGRATED` — see :func:`do_migration`'s docstring for why this is scoped to the + mapping/identity side only, not gnomAD/ClinVar/VEP. + + The caller only accumulates these bare ids (not the annotation payload itself) into a global, + whole-run set — a dedup key is all a deferred, allele-batched annotation pass + (:func:`_backfill_annotation_timelines`) needs to know which alleles to revisit; the actual + gnomAD/ClinVar/VEP observations are re-read fresh from the DB per allele-batch there, not carried + in memory from here. Carrying full ``_AnnotationObservation`` payloads for the whole run was the + previous design and is what overloaded the heap at scale — see that function's docstring. Returns + an empty set for a variant with no current version at all — that one is a full skip, since there is + no live state to reconstruct. + + A version with no resolvable sequence level is skipped **individually**, not the whole variant: + ``ValidTime`` history is exactly the mechanism for retaining partial provenance, so one bad version + should not cost its siblings their records. A window's ``valid_from``/``valid_to`` only depend on + its neighbors' ``mapped_date`` (computed up front in :func:`_reconstruct_windows`), never on + whether a record actually got created for those neighbors — so skipping a window here doesn't + distort the boundaries around it. Already-migrated windows (from a prior partial run that + stopped here before this fix, or a live-level failure that keeps this variant eligible for retry — + see :func:`_variant_ids_to_migrate`) are detected via ``existing_records`` (bulk pre-fetched once per + chunk by the caller — see :func:`do_migration`) and skipped as a no-op, so a re-run is safe to + resume mid-variant. Their annotations are not re-derived here; run ``rebuild-annotations`` after a + resumed run to backfill any timelines missed by the skip. + + Every ``MappingRecord``/``Allele``/``MappingRecordAllele`` this variant needs is only ``db.add``'d + here, not flushed, until one ``db.flush()`` at the end covering the whole variant — cut from one + flush per *window* (this was the dominant cost at scale: round trips, not query time, were the + bottleneck). ``MappingRecordAllele`` is built from the ``record``/``allele`` *objects*, not their + ``.id``, specifically so it doesn't need them flushed first — SQLAlchemy resolves the FKs at flush + time. Observations and ``MIGRATED`` events (which need real ``allele.id``/``variant_id`` values) are + built in a second pass over ``pending`` *after* that single flush, once ids exist. + + Skips/anomalies are recorded into ``skipped`` (reason → variant ids), not logged per variant: a + single unmappable score set (e.g. a large pairwise-haplotype library the mapper never converted) + can contribute hundreds of thousands of skips, so :func:`do_migration` rolls them up into one line + per score set at the end rather than emitting a warning each. + """ + result = _reconstruct_windows(mvs) + if result is None: + stats["variants_skipped_no_current"] += 1 + skipped["no_current"].append(variant_id) + logger.debug("Variant %s has mapped_variants but none current; skipped.", variant_id) + return set() + + if result.live_precedes_history: + # Not fatal — the clamp in _reconstruct_windows keeps windows non-inverted — but it silently + # collapses a historical window to zero-width, so surface it loudly: this shape means the + # live mapped_variant's mapped_date is older than a superseded one, which shouldn't happen + # absent a source data-quality issue, and likely wants a re-map to fix at the source. + stats["variants_live_precedes_history"] += 1 + skipped["live_precedes_history"].append(variant_id) + + # (mv, record, allele-or-None, hgvs_assay_level, valid_from, valid_to) per window actually created + # this pass — resolved to real ids by one flush below, then walked again to emit events and collect + # touched allele ids. + pending: list[tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]]] = [] + + for mv, valid_from, valid_to in result.windows: + if (variant_id, mv.mapped_date) in existing_records: + continue # already migrated in a prior pass over this variant; resuming, not redoing + + level = _resolve_level(mv) + if level is None: + is_live = valid_to is None + reason = "no_level_live" if is_live else "no_level_historical" + stats[f"windows_skipped_{reason}"] += 1 + skipped[reason].append(variant_id) + logger.debug( + "Variant %s has an unattributable mapped_variant (id=%s, live=%s); window skipped.", + variant_id, + mv.id, + is_live, + ) + continue + + hgvs_assay_level, hgvs_source = _hgvs_assay_level_with_source(mv, level) + if hgvs_source == "level_column": + stats["hgvs_recovered_from_level_column"] += 1 + elif hgvs_source == "vrs_expression": + stats["hgvs_recovered_from_vrs_expression"] += 1 + + record = MappingRecord( + variant_id=variant_id, + vrs_digest=(mv.pre_mapped or {}).get("id"), + pre_mapped=mv.pre_mapped or None, + assay_level=level, + hgvs_assay_level=hgvs_assay_level, + mapped_date=mv.mapped_date, + vrs_version=mv.vrs_version, + mapping_api_version=mv.mapping_api_version, + target_gene_mapping_id=mv.target_gene_mapping_id, + alignment_level=mv.alignment_level, + at_mismatched_locus=mv.at_mismatched_locus, + near_gap=mv.near_gap, + valid_from=valid_from, + valid_to=valid_to, + ) + db.add(record) + stats["records"] += 1 + + allele = _authoritative_allele(db, mv, level, allele_cache) + if allele is not None: + db.add( + MappingRecordAllele( + mapping_record=record, + allele=allele, + is_authoritative=True, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["record_alleles"] += 1 + + pending.append((mv, record, allele, hgvs_assay_level, valid_from, valid_to)) + + if not pending: + return set() + + db.flush() # one round trip resolves every record/allele/link id added for this whole variant + stats["variants"] += 1 + + touched_allele_ids: set[int] = set() + for mv, record, allele, hgvs_assay_level, valid_from, valid_to in pending: + # VRS_MAPPING: variant-subject, written for every version regardless of allele — mirrors the + # live job (mapping.py), whose FAILED/ABSENT outcomes get a record but no allele too. + if mv.error_message: + mapping_disposition = Disposition.FAILED + elif allele is not None: + mapping_disposition = Disposition.PRESENT + else: + mapping_disposition = Disposition.ABSENT + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant_id, + disposition=mapping_disposition, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 1 + + if allele is not None: + # CLINGEN_ALLELE_ID / MAPPED_HGVS: allele-subject, only meaningful once an allele exists. + # allele.id is real now — this loop runs after the single flush above. + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT if mv.clingen_allele_id else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + annotation_manager.record_event( + AnnotationType.MAPPED_HGVS, + allele_id=allele.id, + disposition=Disposition.PRESENT if hgvs_assay_level else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 2 + + touched_allele_ids.add(allele.id) + + # Flushed once for the whole variant (not per event) — still inside this variant's own SAVEPOINT + # (the caller's `with db.begin_nested():` wraps this entire call), so a buffered event never + # outlives the rollback of the record/allele it references. See do_migration's except-block for the + # backstop: if something *between* here and there still raised, pending events for this variant are + # explicitly discarded rather than leaking into a later, unrelated flush. + annotation_manager.flush() + return touched_allele_ids + + +def _variant_ids_to_migrate(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Variant ids that have ``mapped_variants`` but no *live* ``MappingRecord`` yet. + + Keyed on a live record specifically, not "any record at all": a variant can have historical + records from a prior pass but still be missing its live one (the live version's mapped_variant had + no resolvable sequence level — see :func:`_migrate_variant`), and that variant must stay eligible + for retry until a re-map (or a QC fix) lets its live window resolve. The parallel-tables invariant + still makes "has mapped_variants and no live mapping record" the exact set of variants with + incomplete substrate; native new-model variants have no ``mapped_variants``. + """ + has_live_record = sa.select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None) + ) + query = ( + sa.select(Variant.id) + .join(MappedVariant, MappedVariant.variant_id == Variant.id) + .where(~has_live_record.exists()) + ) + if score_set_urn is not None: + query = query.join(ScoreSet, ScoreSet.id == Variant.score_set_id).where(ScoreSet.urn == score_set_urn) + return list(db.scalars(query.distinct().order_by(Variant.id)).all()) + + +_SKIP_REASONS = { + "no_current": "mapped_variants present but none current", + "no_level_historical": "a superseded mapped_variant with no resolvable sequence level (window skipped, siblings rescued)", + "no_level_live": "the LIVE mapped_variant has no resolvable sequence level — no live record could be created; remains eligible for retry", + "live_precedes_history": "the live mapped_variant's mapped_date precedes a historical one (data anomaly — a re-map candidate)", +} + + +def _log_skip_summary(db: Session, skipped: dict[str, list[int]], *, chunk: int = 5000) -> None: + """Roll skipped variants up into one log line per (reason, score set). + + Skips cluster hard by score set — a whole library the mapper couldn't convert skips en masse — so + a per-variant warning drowns the log while a per-score-set count pinpoints exactly which set to + chase. Variant ids are resolved to score set URNs in chunks to keep the ``IN`` clauses bounded. + """ + for reason, variant_ids in skipped.items(): + if not variant_ids: + continue + counts: Counter = Counter() + for offset in range(0, len(variant_ids), chunk): + part = variant_ids[offset : offset + chunk] + for urn, n in db.execute( + sa.select(ScoreSet.urn, sa.func.count()) + .join(Variant, Variant.score_set_id == ScoreSet.id) + .where(Variant.id.in_(part)) + .group_by(ScoreSet.urn) + ).all(): + counts[urn] += n + detail = _SKIP_REASONS.get(reason, reason) + logger.warning("Skipped %d variant(s) across %d score set(s) — %s:", len(variant_ids), len(counts), detail) + for urn, n in counts.most_common(): + logger.warning(" %s: %d variant(s) skipped (%s).", urn, n, reason) + + +def do_migration( + db: Session, + *, + score_set_urn: Optional[str] = None, + batch_size: int = 1000, + dry_run: bool = False, + run_at: Optional[datetime] = None, +) -> None: + """Backfill the allele substrate from ``mapped_variants``, committing per batch. + + ``run_at`` (default: now) is the single cutover instant used to force-close any protein-level + ClinVar link that would otherwise land live — see :func:`_write_allele_timeline`. Exposed as a + parameter mainly for deterministic tests; a real run just wants "now". + + Also backfills the ``annotation_event`` audit log — but only for the mapping/identity side + (``VRS_MAPPING``, ``CLINGEN_ALLELE_ID``, ``MAPPED_HGVS``), not gnomAD/ClinVar/VEP. That scope split + is deliberate: the mapping side is a genuine, non-lossy reconstruction (one real historical event + per ``mapped_variants`` version, the same data already trusted enough to build ``MappingRecord``/ + ``Allele`` from), whereas gnomAD/ClinVar/VEP would need to guess at a created/reconfirmed/skipped + cadence legacy data never recorded — and those sources get real, fresh events again once each score + set is re-annotated, so the audit gap there is temporary, not permanent. See + :data:`EventReason.MIGRATED`. + """ + print("Backfilling the Allele substrate from mapped_variants...") + if dry_run: + print(" DRY RUN — no changes will be committed.") + run_at = run_at or datetime.now(timezone.utc) + annotation_manager = AnnotationStatusManager(db) + + variant_ids = _variant_ids_to_migrate(db, score_set_urn) + print(f"Found {len(variant_ids)} variants to migrate.") + + stats: Counter = Counter() + # Wall-clock spent in each phase, so a run reports where the time goes (batch build vs. the + # final annotation pass) and whether the per-batch rate degrades as the tables grow. + phase_seconds: Counter = Counter() + # Ids only — a deduped allele can be shared by variants in different chunks, so its annotation + # timeline can only be assembled once every contributing variant has been migrated, but the actual + # gnomAD/ClinVar/VEP payload doesn't need to ride along in memory to get there: it's re-read fresh + # from the DB per allele-batch in the deferred pass below (_backfill_annotation_timelines). Holding + # just ids here (not full _AnnotationObservation objects) keeps this set's footprint tiny even at + # millions of variants — see that function's docstring for why the old accumulate-everything design + # degraded badly at scale. + touched_allele_ids: set[int] = set() + # Skipped variant ids by reason, rolled up into a per-score-set summary at the end instead of a + # per-variant warning (one unmappable library can skip hundreds of thousands of variants). + skipped: dict[str, list[int]] = defaultdict(list) + + run_start = perf_counter() + for offset in range(0, len(variant_ids), batch_size): + chunk = variant_ids[offset : offset + batch_size] + + mark = perf_counter() + mvs_by_variant: dict[int, list[MappedVariant]] = {vid: [] for vid in chunk} + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(chunk))).all(): + mvs_by_variant[mv.variant_id].append(mv) + phase_seconds["fetch_mapped_variants"] += perf_counter() - mark + + # Bulk pre-fetch, once per chunk instead of once per window — this (not query execution time) + # was the actual bottleneck: a per-window round trip for "does this record already exist" plus + # another for "does this allele already exist" dominates wall-clock at millions-of-rows scale, + # far more than the cost of the queries themselves. + mark = perf_counter() + existing_records: set[tuple[int, date]] = set( + db.execute( + sa.select(MappingRecord.variant_id, MappingRecord.mapped_date).where( + MappingRecord.variant_id.in_(chunk) + ) + ).all() + ) + candidate_digests = { + (mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs + } + candidate_digests.discard(None) + allele_cache: dict[str, Allele] = {} + if candidate_digests: + for allele in db.scalars(sa.select(Allele).where(Allele.vrs_digest.in_(candidate_digests))).all(): + allele_cache[allele.vrs_digest] = allele + phase_seconds["prefetch"] += perf_counter() - mark + + mark = perf_counter() + for variant_id in chunk: + # A per-variant SAVEPOINT so one bad variant rolls back only its own inserts (not the + # whole batch), and its touched allele ids are simply never added rather than referencing + # rolled-back rows. + pending_events_before = len(annotation_manager._pending) + try: + with db.begin_nested(): + variant_touched_allele_ids = _migrate_variant( + db, + variant_id, + mvs_by_variant[variant_id], + stats, + skipped, + annotation_manager, + existing_records, + allele_cache, + ) + except Exception as exc: # noqa: BLE001 — one bad variant must not abort the batch + # Discard any events this variant buffered but never got to flush before raising — they + # would otherwise linger in the manager's queue and get inserted by a later, unrelated + # flush, still referencing this variant/allele's now-rolled-back id (an FK violation far + # from its actual cause). _migrate_variant flushes its own events before returning + # normally, so this is a no-op on the success path. + del annotation_manager._pending[pending_events_before:] + stats["variants_errored"] += 1 + logger.exception("Failed to migrate variant %s: %s", variant_id, exc) + continue + touched_allele_ids.update(variant_touched_allele_ids) + phase_seconds["build_records"] += perf_counter() - mark + + mark = perf_counter() + if dry_run: + db.rollback() + touched_allele_ids.clear() # nothing persisted this batch; these ids are now invalid + else: + db.commit() + phase_seconds["commit"] += perf_counter() - mark + + processed = min(offset + batch_size, len(variant_ids)) + elapsed = perf_counter() - run_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Processed {processed}/{len(variant_ids)} variants... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + if not dry_run and touched_allele_ids: + print(f"Building annotation timelines for {len(touched_allele_ids)} allele(s)...") + mark = perf_counter() + _backfill_annotation_timelines( + db, sorted(touched_allele_ids), batch_size=batch_size, run_at=run_at, stats=stats + ) + phase_seconds["annotation_timelines"] += perf_counter() - mark + + print("\nMigration completed:") + for key in ( + "variants", + "records", + "record_alleles", + "hgvs_recovered_from_level_column", + "hgvs_recovered_from_vrs_expression", + "annotation_events_written", + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + "variants_skipped_no_current", + "windows_skipped_no_level_historical", + "windows_skipped_no_level_live", + "variants_live_precedes_history", + "variants_errored", + ): + print(f" {key}: {stats[key]}") + + _log_skip_summary(db, skipped) + + total = perf_counter() - run_start + print("\nTiming by phase:") + for phase in ("fetch_mapped_variants", "build_records", "commit", "annotation_timelines"): + seconds = phase_seconds[phase] + share = f"{100 * seconds / total:.0f}%" if total else "n/a" + print(f" {phase}: {seconds:.1f}s ({share})") + variants_done = stats["variants"] + overall_rate = f"{variants_done / total:.0f}/s" if total else "n/a" + print(f" total: {total:.1f}s ({overall_rate})") + + +def _allele_ids_to_rebuild(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Every allele id with an authoritative link, optionally scoped to one score set's variants.""" + query = sa.select(sa.distinct(MappingRecordAllele.allele_id)).where(MappingRecordAllele.is_authoritative) + if score_set_urn is not None: + query = ( + query.join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(ScoreSet, ScoreSet.id == Variant.score_set_id) + .where(ScoreSet.urn == score_set_urn) + ) + return list(db.scalars(query.order_by(MappingRecordAllele.allele_id)).all()) + + +def rebuild_annotation_timelines( + db: Session, *, score_set_urn: Optional[str] = None, batch_size: int = 1000, run_at: Optional[datetime] = None +) -> None: + """(Re)build annotation link timelines for already-migrated variants, independently of record + creation. Idempotent (existence-checked, savepoint-isolated). Use to recover after a crash in the + ``migrate`` annotation pass, or to refresh timelines after fixing annotation logic. + + Rebuilds strictly from the legacy proxy data (``mapped_variants`` associations), same as + ``migrate`` — it is not the cross-level fan-out job — so it applies the same protein-level ClinVar + closure rule (see :func:`_write_allele_timeline`) using its own ``run_at`` (default: now). Batches by + allele id via :func:`_backfill_annotation_timelines`, the same bounded-memory pass ``migrate`` itself + uses — this used to accumulate every observation for the whole rebuild in memory before writing + anything, which had the identical unbounded-growth problem the ``migrate`` pass was fixed for. + """ + print("Rebuilding annotation timelines from existing records...") + run_at = run_at or datetime.now(timezone.utc) + allele_ids = _allele_ids_to_rebuild(db, score_set_urn) + print(f"Found {len(allele_ids)} allele(s) with authoritative links.") + + stats: Counter = Counter() + _backfill_annotation_timelines(db, allele_ids, batch_size=batch_size, run_at=run_at, stats=stats) + + print("\nRebuild completed:") + for key in ( + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + ): + print(f" {key}: {stats[key]}") + + +def verify_migration(db: Session, *, score_set_urn: Optional[str] = None) -> None: + """Report substrate coverage without writing.""" + print("\nVerifying backfill...") + + remaining = len(_variant_ids_to_migrate(db, score_set_urn)) + total_mapped_variant_variants = db.scalar(sa.select(sa.func.count(sa.distinct(MappedVariant.variant_id)))) + total_records = db.scalar(sa.select(sa.func.count(MappingRecord.id))) + live_records = db.scalar(sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.valid_to.is_(None))) + total_alleles = db.scalar(sa.select(sa.func.count(Allele.id))) + live_auth_links = db.scalar( + sa.select(sa.func.count(MappingRecordAllele.id)).where( + MappingRecordAllele.is_authoritative, MappingRecordAllele.valid_to.is_(None) + ) + ) + gnomad_links = db.scalar(sa.select(sa.func.count(GnomadAlleleLink.id))) + clinvar_links = db.scalar(sa.select(sa.func.count(ClinvarAlleleLink.id))) + vep_consequences = db.scalar(sa.select(sa.func.count(VepAlleleConsequence.id))) + migrated_events = db.scalar( + sa.select(sa.func.count(AnnotationEvent.id)).where(AnnotationEvent.reason == EventReason.MIGRATED.value) + ) + + print(f" Variants with mapped_variants: {total_mapped_variant_variants}") + print(f" Variants still needing backfill: {remaining}") + print(f" MappingRecords (total / live): {total_records} / {live_records}") + print(f" Alleles (deduped): {total_alleles}") + print(f" Live authoritative links: {live_auth_links}") + print(f" gnomAD / ClinVar / VEP annotations: {gnomad_links} / {clinvar_links} / {vep_consequences}") + print(f" MIGRATED annotation_event rows: {migrated_events}") + if remaining == 0: + print(" ✓ Every legacy variant has a MappingRecord.") + else: + print(f" ⚠ {remaining} variants remain (see logged anomalies, or re-run migrate).") + + +def rollback_migration(db: Session) -> None: + """Delete backfilled ``MappingRecord``s (cascading their allele links). + + Scoped to variants that also have ``mapped_variants`` — the exact discriminator + for backfilled records, since native new-model variants have no legacy rows. + Deduped ``Allele``s and annotation links are left in place: alleles are content + -addressed and harmless if orphaned (a re-run reuses them), and annotation + links re-existence-check. Delete those manually if a full teardown is needed. + + ``AnnotationEvent`` rows (reason ``MIGRATED``) are left in place too, on purpose: it is an + append-only audit log, and deleting them on rollback would erase the very audit trail it exists to + keep — a re-run after rollback just appends a fresh set (matching how ``MappingRecord`` itself gets + deleted and recreated), not a duplicate-avoidance concern the way the ValidTime domain tables are. + """ + print("Rolling back backfilled MappingRecords (scoped to variants with mapped_variants)...") + legacy_variant_ids = sa.select(sa.distinct(MappedVariant.variant_id)) + count = db.scalar( + sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.variant_id.in_(legacy_variant_ids)) + ) + # ON DELETE CASCADE on mapping_record_alleles.mapping_record_id removes the links. + db.execute(sa.delete(MappingRecord).where(MappingRecord.variant_id.in_(legacy_variant_ids))) + db.commit() + print(f"Deleted {count} MappingRecords (and their allele links via cascade).") + print("Note: deduped Alleles, annotation links, and MIGRATED annotation_event rows were left in place") + print("(re-runnable / harmless orphans; the event log is append-only by design).") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Backfill the Allele substrate from mapped_variants.") + parser.add_argument( + "command", nargs="?", default="migrate", choices=["migrate", "verify", "rollback", "rebuild-annotations"] + ) + parser.add_argument("--score-set", dest="score_set_urn", default=None, help="Limit to one score set URN.") + parser.add_argument("--batch-size", type=int, default=1000, help="Variants per commit (default 1000).") + parser.add_argument("--dry-run", action="store_true", help="Run without committing (rolls back each batch).") + return parser.parse_args() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args = _parse_args() + + if args.command == "rollback": + print("WARNING: this deletes all backfilled MappingRecords and their allele links.") + if input("Are you sure you want to continue? (y/N): ").lower() == "y": + with SessionLocal() as session: + rollback_migration(session) + else: + print("Rollback cancelled.") + elif args.command == "verify": + with SessionLocal() as session: + verify_migration(session, score_set_urn=args.score_set_urn) + elif args.command == "rebuild-annotations": + with SessionLocal() as session: + rebuild_annotation_timelines(session, score_set_urn=args.score_set_urn, batch_size=args.batch_size) + verify_migration(session, score_set_urn=args.score_set_urn) + else: + with SessionLocal() as session: + do_migration( + session, + score_set_urn=args.score_set_urn, + batch_size=args.batch_size, + dry_run=args.dry_run, + ) + verify_migration(session, score_set_urn=args.score_set_urn) diff --git a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py index 4e793ffe5..41eec4665 100644 --- a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py +++ b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py @@ -76,7 +76,7 @@ from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import from mavedb.db.session import SessionLocal from mavedb.lib.variants import HGVS_G_REGEX, HGVS_P_REGEX, get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene @@ -102,18 +102,18 @@ _NUCLEOTIDE_ALPHABET = frozenset("ACGTUNRYSWKMBDHV-") -def _layer_from_hgvs(hgvs: str) -> Optional[AnnotationLayer]: - """Classify an HGVS string into an :class:`AnnotationLayer`.""" +def _layer_from_hgvs(hgvs: str) -> Optional[SequenceLevel]: + """Classify an HGVS string into an :class:`SequenceLevel`.""" if HGVS_G_REGEX.search(hgvs): - return AnnotationLayer.genomic + return SequenceLevel.genomic if HGVS_P_REGEX.search(hgvs): - return AnnotationLayer.protein + return SequenceLevel.protein if _HGVS_C_REGEX.search(hgvs): - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[AnnotationLayer]: +def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[SequenceLevel]: """ Infer layer from VRS sequence alphabet (protein vs nucleotide). Returns protein if any allele/member sequence contains a non-nucleotide letter. @@ -150,15 +150,15 @@ def extract_sequences(obj): for seq in extract_sequences(post_mapped): found_seq = True if any(letter.upper() not in _NUCLEOTIDE_ALPHABET for letter in seq if letter.isalpha()): - return AnnotationLayer.protein + return SequenceLevel.protein if found_seq: - return AnnotationLayer.genomic + return SequenceLevel.genomic return None -def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: +def _alignment_level_for(target: TargetGene) -> Optional[SequenceLevel]: """Recover the alignment layer for legacy mapped variants of ``target``. Tries ``post_mapped_metadata`` first, then ``pre_mapped_metadata`` (same @@ -177,16 +177,16 @@ def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: return None if has_genomic: - return AnnotationLayer.genomic + return SequenceLevel.genomic if has_protein: - return AnnotationLayer.protein + return SequenceLevel.protein if has_cdna: - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: +def _layer_from_sequence_type(target: TargetGene) -> Optional[SequenceLevel]: """Infer alignment layer from ``target_sequence.sequence_type`` and ``category``. Last-resort fallback for targets where all variant-level mappings failed @@ -201,16 +201,16 @@ def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: return None seq_type = target.target_sequence.sequence_type if seq_type == "protein": - return AnnotationLayer.protein + return SequenceLevel.protein if seq_type == "dna" and target.category is not None and target.category.value == "protein_coding": - return AnnotationLayer.genomic + return SequenceLevel.genomic return None def _populate_layer_by_target_id( db: Session, targets_by_score_set_id: dict[int, list[TargetGene]], - layer_by_target_id: dict[int, Optional[AnnotationLayer]], + layer_by_target_id: dict[int, Optional[SequenceLevel]], layer_via_hgvs_target_ids: set[int], layer_via_sequence_type_target_ids: set[int], ) -> None: @@ -344,7 +344,7 @@ def do_migration(db: Session) -> None: # Cache (target_gene_id, alignment_level, tool_version) -> persisted TargetGeneMapping # so we create one row per distinct combination and reuse it across every # mapped variant that maps into it. - cache: dict[tuple[int, AnnotationLayer, str], TargetGeneMapping] = {} + cache: dict[tuple[int, SequenceLevel, str], TargetGeneMapping] = {} total = db.scalar(sa.select(sa.func.count(MappedVariant.id))) or 0 print(f" {total} mapped variants to consider.") @@ -393,7 +393,7 @@ def do_migration(db: Session) -> None: # only need to determine it once per target -- and any single attributable mapped # variant for that target lets us attribute every sibling. The via_* sets track which # fallback was used so the diagnostic report shows attribution breadth. - layer_by_target_id: dict[int, Optional[AnnotationLayer]] = {} + layer_by_target_id: dict[int, Optional[SequenceLevel]] = {} layer_via_hgvs_target_ids: set[int] = set() layer_via_sequence_type_target_ids: set[int] = set() _populate_layer_by_target_id( @@ -420,7 +420,7 @@ def do_migration(db: Session) -> None: # Group ids to update by (tgm_id, level) so we can issue one bulk UPDATE # per group instead of a per-row ORM flush. - updates_by_group: dict[tuple[int, AnnotationLayer], list[int]] = defaultdict(list) + updates_by_group: dict[tuple[int, SequenceLevel], list[int]] = defaultdict(list) for row in chunk: last_id = row.id diff --git a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py index 479c51a33..1ce888933 100644 --- a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py +++ b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py @@ -8,8 +8,6 @@ from alembic import op -from mavedb.models.published_variant import signature - # revision identifiers, used by Alembic. revision = "4726e4dddde8" @@ -17,6 +15,11 @@ branch_labels = None depends_on = None +# Frozen literal, not imported from the model: this migration replaces the historical +# mapped_variants-based MV indexes at its point in history. A later migration rebuilds the MV onto +# the mapping_records substrate and re-indexes it, so the column names below stay the legacy ones. +signature = "published_variants_materialized_view" + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### diff --git a/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py new file mode 100644 index 000000000..6c1fc1d99 --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py @@ -0,0 +1,151 @@ +"""Add mapping_records, alleles, and mapping_record_alleles tables + +Revision ID: a1b2c3d4e5f6 +Revises: a7f3c2e9b104 +Create Date: 2026-05-29 + +New parallel tables for the Better Reverse Translation epic (#746). +The existing mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "a7f3c2e9b104" +branch_labels = None +depends_on = None + +VALID_ASSAY_LEVELS = "('genomic', 'cdna', 'protein')" +VALID_ALIGNMENT_LEVELS = "('protein', 'cdna', 'genomic')" + + +def upgrade() -> None: + op.create_table( + "alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=False), + sa.Column("level", sa.String(length=16), nullable=False), + sa.Column("transcript", sa.String(), nullable=False), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("clingen_allele_id", sa.String(), nullable=True), + sa.Column("post_mapped", postgresql.JSONB(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + ) + op.create_index("ix_alleles_vrs_digest", "alleles", ["vrs_digest"]) + op.create_index("ix_alleles_level", "alleles", ["level"]) + op.create_index("ix_alleles_clingen_allele_id", "alleles", ["clingen_allele_id"]) + + op.create_table( + "mapping_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=True), + sa.Column("pre_mapped", postgresql.JSONB(), nullable=True), + sa.Column("assay_level", sa.String(length=16), nullable=False), + sa.Column("hgvs_assay_level", sa.String(), nullable=True), + sa.Column("mapping_api_version", sa.String(), nullable=False), + sa.Column("mapped_date", sa.Date(), nullable=False), + sa.Column("vrs_version", sa.String(), nullable=True), + sa.Column("current", sa.Boolean(), nullable=False), + sa.Column("alignment_level", sa.String(length=16), nullable=True), + sa.Column("at_mismatched_locus", sa.Boolean(), nullable=True), + sa.Column("near_gap", sa.Boolean(), nullable=True), + sa.Column("target_gene_mapping_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.ForeignKeyConstraint( + ["variant_id"], + ["variants.id"], + name="fk_mapping_records_variant_id", + ), + sa.ForeignKeyConstraint( + ["target_gene_mapping_id"], + ["target_gene_mappings.id"], + name="fk_mapping_records_target_gene_mapping_id", + ), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + f"assay_level IN {VALID_ASSAY_LEVELS}", + name="ck_mapping_records_assay_level_valid", + ), + ) + op.create_index("ix_mapping_records_variant_id", "mapping_records", ["variant_id"]) + op.create_index("ix_mapping_records_vrs_digest", "mapping_records", ["vrs_digest"]) + op.create_index( + "ix_mapping_records_target_gene_mapping_id", + "mapping_records", + ["target_gene_mapping_id"], + ) + + op.create_table( + "mapping_record_alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column( + "is_authoritative", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.ForeignKeyConstraint( + ["mapping_record_id"], + ["mapping_records.id"], + name="fk_mapping_record_alleles_mapping_record_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_mapping_record_alleles_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + ["mapping_record_id"], + ) + op.create_index( + "ix_mapping_record_alleles_allele_id", + "mapping_record_alleles", + ["allele_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_allele_id", table_name="mapping_record_alleles") + op.drop_index("ix_mapping_record_alleles_mapping_record_id", table_name="mapping_record_alleles") + op.drop_table("mapping_record_alleles") + + op.drop_index("ix_mapping_records_target_gene_mapping_id", table_name="mapping_records") + op.drop_index("ix_mapping_records_vrs_digest", table_name="mapping_records") + op.drop_index("ix_mapping_records_variant_id", table_name="mapping_records") + op.drop_table("mapping_records") + + op.drop_index("ix_alleles_clingen_allele_id", table_name="alleles") + op.drop_index("ix_alleles_level", table_name="alleles") + op.drop_index("ix_alleles_vrs_digest", table_name="alleles") + op.drop_table("alleles") diff --git a/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py new file mode 100644 index 000000000..c8f702d72 --- /dev/null +++ b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py @@ -0,0 +1,91 @@ +"""rewrite v_variant_annotations onto the mapping_records/alleles substrate + +Revision ID: a1d4f7c2e9b0 +Revises: e7b2c9a1f4d3 +Create Date: 2026-07-18 + +Drops the legacy ``mapped_variants``-based ``v_variant_annotations`` view and recreates it over the +``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate (part of the MappedVariant +read-path teardown). Column changes vs. the legacy shape: + * ``mapped_variant_id`` -> ``mapping_record_id`` (the mapping identity is now the mapping record). + * ``mapping_error`` dropped — there is no per-mapping-record error column; per-subject annotation + dispositions live in ``v_current_annotation_events`` and score-set-level mapping errors on + ``scoresets.mapping_errors``. + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``variant_annotation_view.definition`` would +couple this fixed revision to mutable code, so the next model edit (the following view rewrite) would +make a replay of *this* revision build that later shape — against tables that may not exist yet at this +point in history — and fail. ``_UPGRADE_DEFINITION`` below was compiled once from the model at +authoring time (``CreateView(signature, definition).compile(postgresql.dialect())``) and must not be +edited to track the model; the next shape change gets its own migration. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1d4f7c2e9b0" +down_revision = "e7b2c9a1f4d3" +branch_labels = None +depends_on = None + +SIGNATURE = "v_variant_annotations" + +# New (mapping_records/alleles) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. Allele-derived columns are correlated scalar +# subqueries so the grain stays one row per (variant, annotation_type), matching the legacy view. +_UPGRADE_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapping_records.id AS mapping_record_id, (SELECT alleles.clingen_allele_id +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS clingen_allele_id, mapping_records.hgvs_assay_level, (SELECT alleles.hgvs_g +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'genomic' + LIMIT 1) AS hgvs_g, (SELECT alleles.hgvs_c +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'cdna' + LIMIT 1) AS hgvs_c, (SELECT alleles.hgvs_p +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'protein' + LIMIT 1) AS hgvs_p, (SELECT vep_allele_consequences.functional_consequence +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_functional_consequence, (SELECT vep_allele_consequences.access_date +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_access_date, mapping_records.mapped_date, mapping_records.mapping_api_version, mapping_records.vrs_version, variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapping_records ON mapping_records.variant_id = variants.id AND mapping_records.valid_to IS NULL + LEFT OUTER JOIN variant_annotation_status ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + +# Legacy (mapped_variants-based) definition, inlined so the downgrade can restore the prior shape. +# ``mapped_variants`` still exists at this revision (dropped in a later teardown migration), so this +# remains runnable on downgrade. +_LEGACY_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + + +def upgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_UPGRADE_DEFINITION}") + + +def downgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_LEGACY_DEFINITION}") diff --git a/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py new file mode 100644 index 000000000..8b655f625 --- /dev/null +++ b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py @@ -0,0 +1,65 @@ +"""add vep_allele_consequences table + +Revision ID: a7c4e9d2f1b8 +Revises: e5f7a9c1b3d4 +Create Date: 2026-06-22 + +New valid-time table holding a deduplicated allele's VEP functional consequence, replacing the frozen +vep_functional_consequence/vep_access_date columns on mapped_variants for new-model writes (Step 2 of +the annotation infrastructure migration, docs/design/annotation-infrastructure-migration.md). A row is +live while valid_to is NULL; the partial unique index enforces a single live consequence per allele +(VEP's most-severe consequence is one current value, so a change supersedes rather than accumulates). +functional_consequence is nullable (reserved for a future negative cache). source_version is the +Ensembl release the consequence was resolved under (coordinated software + transcript set + vocabulary), +which version-keys the refresh skip like gnomAD's db_version; access_date is retained as a "last +confirmed" audit stamp. The VEP columns on mapped_variants are left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c4e9d2f1b8" +down_revision = "e5f7a9c1b3d4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "vep_allele_consequences", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("functional_consequence", sa.String(), nullable=True), + sa.Column("source_version", sa.String(), nullable=False), + sa.Column("access_date", sa.Date(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_vep_allele_consequences_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_vep_allele_consequences_allele_id", + "vep_allele_consequences", + ["allele_id"], + ) + # One live consequence per allele: VEP's most-severe consequence is a single current value, so a + # changed result supersedes the prior row rather than accumulating one live row per access. + op.create_index( + "uq_vep_allele_consequences_live", + "vep_allele_consequences", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_vep_allele_consequences_live", table_name="vep_allele_consequences") + op.drop_index("ix_vep_allele_consequences_allele_id", table_name="vep_allele_consequences") + op.drop_table("vep_allele_consequences") diff --git a/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py new file mode 100644 index 000000000..cf8d77e35 --- /dev/null +++ b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py @@ -0,0 +1,79 @@ +"""add annotation_event log + +Revision ID: a7e1c4f9b3d2 +Revises: d4e5f6a7b8c9 +Create Date: 2026-06-25 16:00:00.000000 + +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7e1c4f9b3d2" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + +_VARIANT_SUBJECT_TYPES = "'vrs_mapping', 'cross_level_translation', 'variant_translation', 'ldh_submission'" +_ALLELE_SUBJECT_TYPES = ( + "'clingen_allele_id', 'gnomad_allele_frequency', 'vep_functional_consequence', 'clinvar_control', 'mapped_hgvs'" +) + + +def upgrade(): + op.create_table( + "annotation_event", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("annotation_type", sa.String(length=50), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=True), + sa.Column("allele_id", sa.Integer(), nullable=True), + sa.Column("disposition", sa.String(length=50), nullable=False), + sa.Column("reason", sa.String(length=50), nullable=False), + sa.Column("source_version", sa.String(length=50), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("job_run_id", sa.Integer(), nullable=True), + sa.Column("score_set_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.CheckConstraint( + f"(annotation_type IN ({_VARIANT_SUBJECT_TYPES}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_ALLELE_SUBJECT_TYPES}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["allele_id"], ["alleles.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["job_run_id"], ["job_runs.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["score_set_id"], ["scoresets.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_annotation_event_allele_type_id", + "annotation_event", + ["allele_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_variant_type_id", + "annotation_event", + ["variant_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_allele_type_version", + "annotation_event", + ["allele_id", "annotation_type", "source_version"], + unique=False, + ) + op.create_index("ix_annotation_event_job_run_id", "annotation_event", ["job_run_id"], unique=False) + + +def downgrade(): + op.drop_index("ix_annotation_event_job_run_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_version", table_name="annotation_event") + op.drop_index("ix_annotation_event_variant_type_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_id", table_name="annotation_event") + op.drop_table("annotation_event") diff --git a/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py new file mode 100644 index 000000000..19cca0789 --- /dev/null +++ b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py @@ -0,0 +1,39 @@ +"""rename clinical_controls to clinvar_controls + +Revision ID: b2c3d4e5f6a7 +Revises: a7c4e9d2f1b8 +Create Date: 2026-06-22 + +Renames the clinical_controls entity table to clinvar_controls, and renames the unique +constraint to match. The frozen association table (mapped_variants_clinical_controls) +and its FK to the renamed table are left structurally intact — PostgreSQL updates the FK +target automatically on table rename. The Python model is renamed ClinicalControl → +ClinvarControl in the same changeset (no data migration). +""" + +from alembic import op + +revision = "b2c3d4e5f6a7" +down_revision = "a7c4e9d2f1b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.rename_table("clinical_controls", "clinvar_controls") + # PostgreSQL does not auto-rename constraints on table rename; rename explicitly so the + # on_conflict_do_update(constraint=...) in the job references the correct name. + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinical_controls_db_name_identifier_version " + "TO uq_clinvar_controls_db_name_identifier_version" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinvar_controls_db_name_identifier_version " + "TO uq_clinical_controls_db_name_identifier_version" + ) + op.rename_table("clinvar_controls", "clinical_controls") diff --git a/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py new file mode 100644 index 000000000..2f9dad721 --- /dev/null +++ b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py @@ -0,0 +1,86 @@ +"""rebuild published_variants MV onto the mapping_records substrate + +Revision ID: b2e5a8d3f0c1 +Revises: a1d4f7c2e9b0 +Create Date: 2026-07-18 + +Rebuilds the ``published_variants_materialized_view`` off the legacy ``mapped_variants`` join onto the +``mapping_records`` substrate (part of the MappedVariant read-path teardown). The MV keeps its grain +(one row per published variant per mapping-record version, unmapped variants retained with a NULL +mapping id) and its published filter. Column changes: + * ``mapped_variant_id`` -> ``mapping_record_id`` + * ``current_mapped_variant`` -> ``current_mapping_record`` (``valid_to IS NULL``) + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``published_variant.definition`` would couple +this fixed revision to mutable code, so a later model edit would make a replay of *this* revision build +that later shape (possibly against tables that no longer exist by then) and fail. ``_UPGRADE_DEFINITION`` +below was compiled once from the model at authoring time and must not be edited to track the model. +""" + +from alembic_utils.pg_materialized_view import PGMaterializedView + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b2e5a8d3f0c1" +down_revision = "a1d4f7c2e9b0" +branch_labels = None +depends_on = None + +SIGNATURE = "published_variants_materialized_view" + +# New (mapping_records-based) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. The LEFT OUTER JOIN to mapping_records carries no +# liveness filter on purpose — every record version is retained and current-ness is exposed via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). +_UPGRADE_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapping_records.id AS mapping_record_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapping_records.valid_to IS NULL AS current_mapping_record +FROM variants LEFT OUTER JOIN mapping_records ON variants.id = mapping_records.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + +# Legacy (mapped_variants-based) definition, inlined for the downgrade path. ``mapped_variants`` still +# exists at this revision (dropped in a later teardown migration), so downgrade remains runnable. +_LEGACY_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + + +def upgrade(): + # Dropping the MV cascades to its owned indexes (the legacy idx_..._ids on mapped_variant_id). + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapping_record_id", "variant_id", "score_set_id"], + unique=True, + ) + + +def downgrade(): + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapped_variant_id", "variant_id", "score_set_id"], + unique=True, + ) diff --git a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py index 7ce51c88d..3b3855e7a 100644 --- a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py +++ b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py @@ -2,13 +2,16 @@ Revision ID: b85bc7b1bec7 Revises: c404b6719110 Create Date: 2025-03-14 01:53:19.898198 + +The MV SQL is inlined as a frozen literal rather than imported from the model. At this revision the +MV joins the (now legacy) ``mapped_variants`` table; a later migration rebuilds it onto the +``mapping_records`` substrate. Importing the live model definition here would make a fresh replay +build *that* later shape at this early revision — before the substrate tables exist — and fail. +Freezing the historical SQL keeps replay green; the rebuild happens in its own migration. """ from alembic import op from alembic_utils.pg_materialized_view import PGMaterializedView -from sqlalchemy.dialects import postgresql - -from mavedb.models.published_variant import signature, definition # revision identifiers, used by Alembic. @@ -17,59 +20,71 @@ branch_labels = None depends_on = None +SIGNATURE = "published_variants_materialized_view" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + def upgrade(): op.create_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) op.create_index( - f"idx_{signature}_variant_id", - signature, + f"idx_{SIGNATURE}_variant_id", + SIGNATURE, ["variant_id"], unique=False, ) op.create_index( - f"idx_{signature}_variant_urn", - signature, + f"idx_{SIGNATURE}_variant_urn", + SIGNATURE, ["variant_urn"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_id", - signature, + f"idx_{SIGNATURE}_score_set_id", + SIGNATURE, ["score_set_id"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_urn", - signature, + f"idx_{SIGNATURE}_score_set_urn", + SIGNATURE, ["score_set_urn"], unique=False, ) op.create_index( - f"idx_{signature}_mapped_variant_id", - signature, + f"idx_{SIGNATURE}_mapped_variant_id", + SIGNATURE, ["mapped_variant_id"], unique=True, ) def downgrade(): - op.drop_index(f"idx_{signature}_variant_id", signature) - op.drop_index(f"idx_{signature}_variant_urn", signature) - op.drop_index(f"idx_{signature}_mapped_variant_id", signature) - op.drop_index(f"idx_{signature}_score_set_id", signature) - op.drop_index(f"idx_{signature}_score_set_urn", signature) + op.drop_index(f"idx_{SIGNATURE}_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_variant_urn", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_mapped_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_urn", SIGNATURE) op.drop_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) diff --git a/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py new file mode 100644 index 000000000..2705d9a3c --- /dev/null +++ b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py @@ -0,0 +1,32 @@ +"""drop alleles.transcript column + +Revision ID: b8e1f0a2c4d7 +Revises: f4d2a9c1b7e3 +Create Date: 2026-06-05 + +The `transcript` column duplicated data already present in the HGVS columns — it was +always extract_accession(hgvs_g/hgvs_c/hgvs_p). It is now a derived hybrid_property on +the Allele model (split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)), so the stored +column is removed to keep a single source of truth and avoid drift. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b8e1f0a2c4d7" +down_revision = "f4d2a9c1b7e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_column("alleles", "transcript") + + +def downgrade() -> None: + # Re-add the column and backfill it from the HGVS columns (the same derivation the + # hybrid_property uses) so the restored NOT NULL column is consistent. + op.add_column("alleles", sa.Column("transcript", sa.String(), nullable=True)) + op.execute("UPDATE alleles SET transcript = split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)") + op.alter_column("alleles", "transcript", nullable=False) diff --git a/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py new file mode 100644 index 000000000..73843866c --- /dev/null +++ b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py @@ -0,0 +1,30 @@ +"""add v_current_annotation_events view + +Revision ID: b8f2c5a1d3e4 +Revises: a7e1c4f9b3d2 +Create Date: 2026-06-26 + +Creates the v_current_annotation_events view: the latest AnnotationEvent per (subject, annotation_type), +with ClinVar partitioned additionally by source_version (multi-live, one current row per release). +Replaces the per-variant v_variant_annotations as the current-state projection over the new +allele-model annotation log. Intended for operator queries, BI, and the annotation CLI scripts. +""" + +from alembic import op + +from mavedb.db.view import CreateView, DropView +from mavedb.models.annotation_event_view import definition, signature + +# revision identifiers, used by Alembic. +revision = "b8f2c5a1d3e4" +down_revision = "a7e1c4f9b3d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(CreateView(signature, definition, materialized=False)) + + +def downgrade() -> None: + op.execute(DropView(signature, materialized=False)) diff --git a/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py new file mode 100644 index 000000000..4a20f008f --- /dev/null +++ b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py @@ -0,0 +1,73 @@ +"""add clinvar_allele_links table + +Revision ID: c3d4e5f6a7b8 +Revises: b2c3d4e5f6a7 +Create Date: 2026-06-22 + +New valid-time link table connecting deduplicated alleles to ClinvarControl rows, replacing +the frozen mapped_variants_clinical_controls association for new-model writes. + +ClinVar's link shape is deliberately multi-live: the partial unique index is +(allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live +link per ClinVar release rather than superseding as in gnomAD/VEP. Each ClinVar release is a +distinct ClinvarControl row, so different releases stack as independent live links. A link is +retired (valid_to closed) only if ClinVar later removes the variant from a release, which +would surface as a re-run finding no data for that release and retiring the corresponding +link — archival data never changes, so this path is theoretical. + +The existing mapped_variants_clinical_controls association table is left untouched (frozen +for serving existing data). +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "c3d4e5f6a7b8" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "clinvar_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("clinvar_control_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_clinvar_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["clinvar_control_id"], + ["clinvar_controls.id"], + name="fk_clinvar_allele_links_clinvar_control_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_clinvar_allele_links_allele_id", "clinvar_allele_links", ["allele_id"]) + op.create_index("ix_clinvar_allele_links_clinvar_control_id", "clinvar_allele_links", ["clinvar_control_id"]) + # Multi-live: one live link per (allele, release). An allele accumulates one live link per + # ClinVar release rather than superseding — unlike gnomAD/VEP which enforce one live link + # per allele across all versions. Superseded rows (valid_to IS NOT NULL) are preserved for + # point-in-time queries. + op.create_index( + "uq_clinvar_allele_links_live", + "clinvar_allele_links", + ["allele_id", "clinvar_control_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_clinvar_allele_links_live", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_clinvar_control_id", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_allele_id", table_name="clinvar_allele_links") + op.drop_table("clinvar_allele_links") diff --git a/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py new file mode 100644 index 000000000..26d3d6dbf --- /dev/null +++ b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py @@ -0,0 +1,48 @@ +"""add valid-time versioning to mapping_record_alleles + +Revision ID: c3d5e7f9a1b2 +Revises: b8e1f0a2c4d7 +Create Date: 2026-06-05 + +Make the link table valid-time versioned (TemporalLink): a link is live while valid_to is +NULL, and superseding it closes valid_to instead of deleting, so reverse translation can be +re-run independently while prior derivations remain queryable point-in-time. The partial +unique index enforces a single live link per (mapping_record, allele). + +Assumes no pre-existing duplicate live links — true for these parallel tables, which are +new-only writes and not yet serving. If this ever runs against data with duplicates, the +unique index creation will fail and the duplicates must be retired first. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c3d5e7f9a1b2" +down_revision = "b8e1f0a2c4d7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.add_column( + "mapping_record_alleles", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "uq_mapping_record_alleles_live", + "mapping_record_alleles", + ["mapping_record_id", "allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live", table_name="mapping_record_alleles") + op.drop_column("mapping_record_alleles", "valid_to") + op.drop_column("mapping_record_alleles", "valid_from") diff --git a/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py new file mode 100644 index 000000000..13c206931 --- /dev/null +++ b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py @@ -0,0 +1,27 @@ +"""index annotation_event.score_set_id + +Revision ID: c9b2a4f8e1d3 +Revises: b8f2c5a1d3e4 +Create Date: 2026-06-30 + +Adds the missing foreign-key index on annotation_event.score_set_id. Every other FK on this table +is indexed; score_set_id was not. The index backs the ON DELETE SET NULL cascade fired when a score +set is deleted (an unindexed FK forces a sequential scan of the event log per deleted score set) and +any operator/BI query that filters the log by score set. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c9b2a4f8e1d3" +down_revision = "b8f2c5a1d3e4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index("ix_annotation_event_score_set_id", "annotation_event", ["score_set_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_annotation_event_score_set_id", table_name="annotation_event") diff --git a/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py new file mode 100644 index 000000000..de5758b4c --- /dev/null +++ b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py @@ -0,0 +1,29 @@ +"""add clinvar_variation_id to clinvar_controls + +Revision ID: d4e5f6a7b8c9 +Revises: c3d4e5f6a7b8 +Create Date: 2026-06-22 + +Additive, non-breaking column for ClinVar's canonical public identifier (VariationID), captured forward +from the variant_summary TSV. db_identifier continues to hold the AlleleID (the allele-level handle used +for gnomAD cross-references); this carries the VariationID beside it for eventual external ClinVar links +(clinvar/variation/{id}). Nullable and not yet served — the dedicated clinvar_variants remodel (explicit +fields replacing the generic db_* shape, the serving/UI cutover, and backfill of existing rows) is +deferred. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c3d4e5f6a7b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("clinvar_controls", sa.Column("clinvar_variation_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("clinvar_controls", "clinvar_variation_id") diff --git a/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py new file mode 100644 index 000000000..603c171a5 --- /dev/null +++ b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py @@ -0,0 +1,76 @@ +"""move mapping_records onto valid-time versioning + +Revision ID: d4e6f8a0b2c3 +Revises: c3d5e7f9a1b2 +Create Date: 2026-06-05 + +Replace the stored `current` flag and the `created_at`/`updated_at` audit dates with valid-time +columns (ValidTime mixin): a mapping record is live while valid_to is NULL, and a re-map retires +the prior version (closing valid_to) instead of flipping a boolean. `current` becomes derived +(valid_to IS NULL). `mapped_date` (the date the mapping was performed) is domain data and stays. + +The partial unique index promotes to the database the "one live mapping record per variant" +invariant the mapping job previously enforced only in app code. + +Backfills from the columns being dropped, so existing rows keep their validity. Assumes no +duplicate live records per variant (true for these pre-cutover parallel tables; otherwise the +unique index creation fails and the duplicates must be retired first). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d4e6f8a0b2c3" +down_revision = "c3d5e7f9a1b2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_records", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True, server_default=sa.func.now()), + ) + op.add_column( + "mapping_records", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + + # Backfill validity from the columns being replaced: a record's life began at created_at, and + # a non-current record was retired at updated_at (the only in-place update it ever took). + op.execute("UPDATE mapping_records SET valid_from = created_at::timestamptz") + op.execute("UPDATE mapping_records SET valid_to = updated_at::timestamptz WHERE current = false") + + op.alter_column("mapping_records", "valid_from", nullable=False) + + op.drop_column("mapping_records", "current") + op.drop_column("mapping_records", "created_at") + op.drop_column("mapping_records", "updated_at") + + op.create_index( + "uq_mapping_records_current", + "mapping_records", + ["variant_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_records_current", table_name="mapping_records") + + op.add_column("mapping_records", sa.Column("current", sa.Boolean(), nullable=True)) + op.add_column("mapping_records", sa.Column("created_at", sa.Date(), nullable=True)) + op.add_column("mapping_records", sa.Column("updated_at", sa.Date(), nullable=True)) + + op.execute("UPDATE mapping_records SET current = (valid_to IS NULL)") + op.execute("UPDATE mapping_records SET created_at = valid_from::date") + op.execute("UPDATE mapping_records SET updated_at = coalesce(valid_to, valid_from)::date") + + op.alter_column("mapping_records", "current", nullable=False) + op.alter_column("mapping_records", "created_at", nullable=False) + op.alter_column("mapping_records", "updated_at", nullable=False) + + op.drop_column("mapping_records", "valid_to") + op.drop_column("mapping_records", "valid_from") diff --git a/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py new file mode 100644 index 000000000..54f3e3d58 --- /dev/null +++ b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py @@ -0,0 +1,71 @@ +"""add gnomad_allele_links table + +Revision ID: e5f7a9c1b3d4 +Revises: d4e6f8a0b2c3 +Create Date: 2026-06-18 + +New valid-time link table connecting deduplicated alleles to gnomAD variants, replacing the frozen +gnomad_variants_mapped_variants association for new-model writes (Step 1 of the annotation +infrastructure migration, docs/design/annotation-infrastructure-migration.md). A link is live while +valid_to is NULL; the partial unique index enforces a single live link per (allele, gnomad variant) +pair. The existing gnomad_variants_mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e5f7a9c1b3d4" +down_revision = "d4e6f8a0b2c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "gnomad_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("gnomad_variant_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_gnomad_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["gnomad_variant_id"], + ["gnomad_variants.id"], + name="fk_gnomad_allele_links_gnomad_variant_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_gnomad_allele_links_allele_id", + "gnomad_allele_links", + ["allele_id"], + ) + op.create_index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_allele_links", + ["gnomad_variant_id"], + ) + # One live link per allele: gnomAD frequency is a single current value, so a version bump + # supersedes the prior link rather than accumulating one live link per version. + op.create_index( + "uq_gnomad_allele_links_live", + "gnomad_allele_links", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_gnomad_allele_links_live", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_gnomad_variant_id", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_allele_id", table_name="gnomad_allele_links") + op.drop_table("gnomad_allele_links") diff --git a/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py new file mode 100644 index 000000000..6d0d0ff7a --- /dev/null +++ b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py @@ -0,0 +1,37 @@ +"""add projection_group to mapping_record_alleles + +Revision ID: e7b2c9a1f4d3 +Revises: f1a3c7e9b2d5 +Create Date: 2026-07-08 + +Adds a nullable ``projection_group`` integer column to ``mapping_record_alleles`` — a within-record +grouping key that pairs the coding and genomic links of one reverse-translation projection pair. +The two members of a pair share a value; the shared protein apex is NULL, as is every row written +before reverse translation runs. See the model and the reverse_translation worker for the full +semantics (per-record index, authoritative fold-in). + +No index is added: the per-URN serving fetch is already covered by +``ix_mapping_record_alleles_mapping_record_id`` and a record's link set is small. No backfill: +existing rows stay NULL and re-group lazily on the next re-map / reverse-translation run. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e7b2c9a1f4d3" +down_revision = "f1a3c7e9b2d5" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("projection_group", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("mapping_record_alleles", "projection_group") diff --git a/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py new file mode 100644 index 000000000..6d7b6c989 --- /dev/null +++ b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py @@ -0,0 +1,46 @@ +"""enforce one live authoritative link per mapping record + +Revision ID: f1a3c7e9b2d5 +Revises: c9b2a4f8e1d3 +Create Date: 2026-07-07 + +Adds a partial unique index enforcing at most one live authoritative link per mapping record. The +existing uq_mapping_record_alleles_live keys on (mapping_record, allele) and so cannot prevent two +different alleles both being flagged is_authoritative for the same record. The serving layer depends +on there being exactly one: the lean whole-set view (lib/score_set_variants) joins the authoritative +link expecting it 1:1 with the variant, so a second live authoritative link would silently duplicate +the variant and double-count its score in the histogram rather than raise. This index moves that +failure to write time in the mapping job, where the bug actually lives. + +Assumes no pre-existing record with two live authoritative links — the invariant is upheld today by +the mapping job's write logic. If this ever runs against data that violates it, the unique index +creation fails and the offending duplicates must be retired first. To check before applying: + + SELECT mapping_record_id, count(*) + FROM mapping_record_alleles + WHERE is_authoritative AND valid_to IS NULL + GROUP BY mapping_record_id HAVING count(*) > 1; +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1a3c7e9b2d5" +down_revision = "c9b2a4f8e1d3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_alleles", + ["mapping_record_id"], + unique=True, + postgresql_where=sa.text("is_authoritative AND valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live_authoritative", table_name="mapping_record_alleles") diff --git a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py index 12ffd574d..2d377def7 100644 --- a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py +++ b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py @@ -7,23 +7,46 @@ Creates the v_variant_annotations convenience view, which joins variants, mapped_variants, variant_annotation_status, and scoresets into a single flat row per (variant, annotation_type). Intended for operator queries and the variant_annotations CLI script. + +The view SQL is inlined as a frozen literal rather than imported from the model. At this point in +history the view joins the (now legacy) ``mapped_variants`` table; a later migration rewrites it onto +the ``mapping_records`` / ``alleles`` substrate. Importing the live model definition here would make a +fresh replay build *that* later shape at this early revision — before the substrate tables exist — +and fail. Freezing the historical SQL keeps replay green; the rewrite happens in its own migration. """ from alembic import op -from mavedb.db.view import CreateView, DropView -from mavedb.models.variant_annotation_view import definition, signature - # revision identifiers, used by Alembic. revision = "f2a1b3c4d5e6" down_revision = "e3a7b9f1d2c5" branch_labels = None depends_on = None +SIGNATURE = "v_variant_annotations" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + def upgrade() -> None: - op.execute(CreateView(signature, definition, materialized=False)) + op.execute(f"CREATE VIEW {SIGNATURE} AS {DEFINITION}") def downgrade() -> None: - op.execute(DropView(signature, materialized=False)) + op.execute(f"DROP VIEW {SIGNATURE}") diff --git a/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py new file mode 100644 index 000000000..790659d08 --- /dev/null +++ b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py @@ -0,0 +1,49 @@ +"""add cross_level_translation annotation type + +Revision ID: f4d2a9c1b7e3 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-02 + +Extends ck_variant_annotation_type_valid to allow the 'cross_level_translation' +annotation type. The VRS mapping worker writes one such row per variant to record +whether cross-level translation (filling the levels the assay did not map) +succeeded, was skipped (multivariant / no transcript), or failed. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f4d2a9c1b7e3" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + +_TYPES_OLD = ( + "'vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) +_TYPES_NEW = "'vrs_mapping', 'cross_level_translation', " + ( + "'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) + + +def upgrade() -> None: + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_NEW})", + ) + + +def downgrade() -> None: + op.execute("DELETE FROM variant_annotation_status WHERE annotation_type = 'cross_level_translation'") + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_OLD})", + ) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9835cff4e..ac4bcff58 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -19,6 +19,7 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ports: - "8002:8000" ulimits: @@ -27,6 +28,7 @@ services: hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo worker: @@ -44,12 +46,14 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ulimits: nofile: soft: 65536 hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo depends_on: - db diff --git a/mypy_stubs/ga4gh/core/__init__.pyi b/mypy_stubs/ga4gh/core/__init__.pyi index f5814d7cf..8044c175f 100644 --- a/mypy_stubs/ga4gh/core/__init__.pyi +++ b/mypy_stubs/ga4gh/core/__init__.pyi @@ -1,3 +1,10 @@ from . import models as models +from .identifiers import PrevVrsVersion as PrevVrsVersion -__all__ = ["models"] +def ga4gh_identify( + vro: object, + in_place: str = ..., + as_version: PrevVrsVersion | None = ..., +) -> str | None: ... + +__all__ = ["PrevVrsVersion", "ga4gh_identify", "models"] diff --git a/mypy_stubs/ga4gh/vrs/dataproxy.pyi b/mypy_stubs/ga4gh/vrs/dataproxy.pyi new file mode 100644 index 000000000..25cabfda9 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/dataproxy.pyi @@ -0,0 +1,8 @@ +from abc import ABC + +class _DataProxy(ABC): + def get_metadata(self, identifier: str) -> dict: ... + def get_sequence(self, identifier: str, start: int | None = ..., end: int | None = ...) -> str: ... + +class SeqRepoDataProxy(_DataProxy): + def __init__(self, sr: object) -> None: ... diff --git a/mypy_stubs/ga4gh/vrs/extras/__init__.pyi b/mypy_stubs/ga4gh/vrs/extras/__init__.pyi new file mode 100644 index 000000000..e69de29bb diff --git a/mypy_stubs/ga4gh/vrs/extras/translator.pyi b/mypy_stubs/ga4gh/vrs/extras/translator.pyi new file mode 100644 index 000000000..d3bb6aeb4 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/extras/translator.pyi @@ -0,0 +1,19 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +class _Translator: + default_assembly_name: str + data_proxy: _DataProxy + identify: bool + def __init__( + self, + data_proxy: _DataProxy, + default_assembly_name: str = ..., + identify: bool = ..., + ) -> None: ... + # Returns a VRS variation (Allele/CisPhasedBlock/...); typed loosely so callers + # can annotate the concrete subtype they expect without a redundant cast. + def translate_from(self, var: str, fmt: str | None = ..., **kwargs: Any) -> Any: ... + +class AlleleTranslator(_Translator): ... diff --git a/mypy_stubs/ga4gh/vrs/normalize.pyi b/mypy_stubs/ga4gh/vrs/normalize.pyi new file mode 100644 index 000000000..1aea15b23 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/normalize.pyi @@ -0,0 +1,5 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +def normalize(vo: Any, data_proxy: _DataProxy | None = ..., **kwargs: Any) -> Any: ... diff --git a/mypy_stubs/hgvs/assemblymapper.pyi b/mypy_stubs/hgvs/assemblymapper.pyi new file mode 100644 index 000000000..798878312 --- /dev/null +++ b/mypy_stubs/hgvs/assemblymapper.pyi @@ -0,0 +1,20 @@ +from typing import Any + +import hgvs.sequencevariant + +class AssemblyMapper: + def __init__( + self, + hdp: Any, + assembly_name: str = ..., + alt_aln_method: str = ..., + normalize: bool = ..., + prevalidation_level: str | None = ..., + in_par_assume: str = ..., + replace_reference: bool = ..., + *args: Any, + **kwargs: Any, + ) -> None: ... + def g_to_t(self, var_g: Any, tx_ac: str) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_g(self, var_c: Any) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_p(self, var_c: Any, translation_table: Any = ...) -> hgvs.sequencevariant.SequenceVariant: ... diff --git a/poetry.lock b/poetry.lock index 99637684e..a7cee0a75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -269,6 +269,51 @@ dev = ["bandit (>=1.7,<2.0)", "build (>=0.8,<1.0)", "flake8 (>=4.0,<5.0)", "ipyt docs = ["mkdocs"] tests = ["pytest (>=7.1,<8.0)", "pytest-cov (>=4.1,<5.0)", "pytest-optional-tests", "tox (>=3.25,<4.0)", "vcrpy"] +[[package]] +name = "biopython" +version = "1.87" +description = "Freely available tools for computational molecular biology." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "biopython-1.87-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:35f13796188412f135acbed196bd6cfedc1257199d50b4883e289bbd96320efc"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63978e1b3ae040c52369bc1bd3f4c668171f5f1f0808798eccd74b575cce455d"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e951f4862ffc1dccc28e1c25245059fa653d86028a88f5dfe1b7875875f3a4f"}, + {file = "biopython-1.87-cp310-cp310-win32.whl", hash = "sha256:86596b05f39afbc5984ee6239c93fd75f6a97fffb9a630d74b45652c24aab964"}, + {file = "biopython-1.87-cp310-cp310-win_amd64.whl", hash = "sha256:efc16dd8a9312eb655fa590821495b0d8ca25d19f7b4ef4fa4da9e71d59d33e5"}, + {file = "biopython-1.87-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:58e36efa7eaa8813cffe440af4824afa20cc3c21b1b179a95cc0fb15c4b83c01"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccf00d15e698656796ab14ff3eb175e282da7a08eedd36a29b6cacec5a33e97f"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bab39ec4108fb2542a9f1012db8269426fd6e86ed84ebc1b0bd11134ef443dd3"}, + {file = "biopython-1.87-cp311-cp311-win32.whl", hash = "sha256:126e18ad44e959a1984560562fa4f295c075ce2e610e8ddbc9ec6f52e09f70d8"}, + {file = "biopython-1.87-cp311-cp311-win_amd64.whl", hash = "sha256:d4ff5369ffb7a966bcf921cae6c8a6eab5070b5df1c9f2ae8c18ad40fdcb7ec5"}, + {file = "biopython-1.87-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:77ccc634621904d4a8fa0a43b5e0f093fa9df8c9577ed3858af648bb3528f51e"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3428155c3e0abbed7aad5ff08e034d435b84dfe560c8ec58e7d43abda4b6a43"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:65ba69ef0273e983a9036c2a228142bc34266179a5f03660fc281d332d718630"}, + {file = "biopython-1.87-cp312-cp312-win32.whl", hash = "sha256:b077777fd2c555434bdcee58743f6f860aa80e1e005d9671913aa73823c6a773"}, + {file = "biopython-1.87-cp312-cp312-win_amd64.whl", hash = "sha256:856e3d64f1f27db493474ff84916ed8572731a525e001c7d0d8f41a0fd187000"}, + {file = "biopython-1.87-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e05ef5d632c319ab3ef77705c74061190d0792b07e1f2b9eee867401b2758e7e"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:772539297fa16a78f38651c793f53f8c11bd18317b111982e72cf30a6e57512a"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d221b2e08e7e89713fdbfb15c8ea6744e908d59f672cd2b6fcf9ed47910d05e"}, + {file = "biopython-1.87-cp313-cp313-win32.whl", hash = "sha256:fab1b12f6bc4646b7f56b4c390ecff685f02b5b29e3a0c10477195bb49fe62f8"}, + {file = "biopython-1.87-cp313-cp313-win_amd64.whl", hash = "sha256:01ee30203bd4b2145cdfe2878499e549a7087f897a6f4d1ebd9de30790123140"}, + {file = "biopython-1.87-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db73fe16aa2b20677ac86d1997612acb0aa1a3720bc899f65d2bce5583208e1b"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ffe272517478691439a59cccd3cc2929fc8f6bfb8cbc8cc5acc103660395a7"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff28a6f31630b3c9f52903478a2ed9dd894b07c1998e40eaeefbeefac20f2d0f"}, + {file = "biopython-1.87-cp314-cp314-win32.whl", hash = "sha256:d740c75d4bc94f9dff51719a0deda37e5e885f06ee6dfbb5e9a21bbe9de35a9c"}, + {file = "biopython-1.87-cp314-cp314-win_amd64.whl", hash = "sha256:98e397096336a49804b6aaaeac8c47ad82e3e4430862f0cde37be73037f1017e"}, + {file = "biopython-1.87-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e4878a9b56775480154c686f81e98f6d907b44d87605bdc2f53538ccdfde9624"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6fcec2e3602ed52ced701f8f7851952383f84dbc4caeb4d202d088170e86b6d"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8da2d44a4b912c7550a051a5ff4bb72a61decc9c4b19ea92cba4c02fffb143c"}, + {file = "biopython-1.87-cp314-cp314t-win32.whl", hash = "sha256:3670d76759c6cb53ba617f9823d3a438c1aa5415abef6addd29cb81d61d7b312"}, + {file = "biopython-1.87-cp314-cp314t-win_amd64.whl", hash = "sha256:331c4151608a1d8406eff0d3c52a0ff1fa3e82604fc85f11c696c562919fb161"}, + {file = "biopython-1.87.tar.gz", hash = "sha256:8456c803459b679a9712422e5a7fd9809f2f089bf69bb085f3b077946ac9bdbf"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "bioutils" version = "0.6.1" @@ -1432,6 +1477,19 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "eutils" version = "0.6.0" @@ -2893,6 +2951,22 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "orcid" version = "1.0.3" @@ -4796,6 +4870,62 @@ dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"] +[[package]] +name = "variant-annotation" +version = "0.1.0" +description = "Genetic variant mapping and annotation pipeline" +optional = true +python-versions = ">=3.11" +groups = ["main"] +markers = "extra == \"server\"" +files = [] +develop = true + +[package.dependencies] +biopython = "*" +boto3 = "*" +click = "*" +openpyxl = "*" +pandas = ">=2.2.1,<3.0.0" +python-dotenv = "*" +redis = "*" +requests = "*" +variant-translation = "1.1.0b1" + +[package.extras] +dev = ["ruff"] +gnomad = ["hail"] +mapping = ["dcd-mapping @ git+https://github.com/VariantEffect/dcd_mapping2.git@v2026.1.2"] +publish = ["build", "twine"] +tests = ["pytest"] + +[package.source] +type = "directory" +url = "../variant-annotation" + +[[package]] +name = "variant-translation" +version = "1.1.0b1" +description = "Reverse-translate protein HGVS variants." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "variant_translation-1.1.0b1-py3-none-any.whl", hash = "sha256:cdf693ab564b767c89e7b463b1345e801d2ab4a5f312631584bf79d178ba6122"}, + {file = "variant_translation-1.1.0b1.tar.gz", hash = "sha256:5151d02ccb5c651c7ba47398d75355f2bd19a36717faa39dc2a390cfb4c7968f"}, +] + +[package.dependencies] +click = "*" +hgvs = "*" +python-dotenv = "*" +requests = "*" + +[package.extras] +dev = ["pytest", "ruff"] +publish = ["build", "twine"] + [[package]] name = "virtualenv" version = "21.3.1" @@ -5098,9 +5228,9 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "variant-annotation", "watchtower"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "270f882db7e5eb3b2ddb642291a0d9234e05e6dd2adeae6ef7a78915f74ef8d5" +content-hash = "615d9e01ee217622e30a81624e5f042b18b2ff5a6d519114e005dcc22fa5cff6" diff --git a/pyproject.toml b/pyproject.toml index f8d0de49c..b2a958be3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ python-dotenv = "^1.2" python-json-logger = "~2.0.7" SQLAlchemy = "~2.0.29" ga4gh-va-spec = "~0.4.2" +ga4gh-cat-vrs = "~0.7.2" setuptools = "~81.0.0" # Optional dependencies for running this application as a server @@ -63,6 +64,7 @@ starlette = { version = "~0.49.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } +variant-annotation = { path = "../variant-annotation", develop = true, optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" @@ -93,7 +95,7 @@ SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" } [tool.poetry.extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "variant-annotation", "watchtower"] [tool.mypy] @@ -104,6 +106,18 @@ plugins = [ ] mypy_path = "mypy_stubs" +# variant-annotation is installed as a PEP 660 editable sibling package whose import +# hook mypy cannot follow, and it ships no py.typed marker. Treat it as untyped rather +# than maintaining drift-prone local stubs for an actively-developed internal library. +[[tool.mypy.overrides]] +module = "variant_annotation.*" +ignore_missing_imports = true + +# ga4gh.cat_vrs (the Cat-VRS 1.0.0 pydantic models) ships no py.typed marker, unlike ga4gh.vrs. +[[tool.mypy.overrides]] +module = "ga4gh.cat_vrs.*" +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index a24dc3167..e91a23354 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7.1" +__version__ = "2026.2.7.1-dev" logger.info(f"MaveDB {__version__}") diff --git a/src/mavedb/data_providers/services.py b/src/mavedb/data_providers/services.py index a94c16d6e..45b91b756 100644 --- a/src/mavedb/data_providers/services.py +++ b/src/mavedb/data_providers/services.py @@ -2,7 +2,9 @@ from typing import TYPE_CHECKING, Optional import boto3 +from biocommons.seqrepo import SeqRepo from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider, SeqFetcher +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from mavedb.lib.mapping import VRSMap @@ -16,6 +18,7 @@ DCD_MAP_URL = os.environ.get("DCD_MAPPING_URL", "http://dcd-mapping:8000") CDOT_URL = os.environ.get("CDOT_URL", "http://cdot-rest:8000") +SEQREPO_DIR = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") CSV_UPLOAD_S3_BUCKET_NAME = os.getenv("UPLOAD_S3_BUCKET_NAME", "score-set-csv-uploads-dev") @@ -27,6 +30,21 @@ def cdot_rest() -> RESTDataProvider: return RESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) +def seqrepo() -> SeqRepo: + return SeqRepo(SEQREPO_DIR) + + +def seqrepo_data_proxy() -> SeqRepoDataProxy: + """VRS sequence/refget data proxy backed by local SeqRepo. + + Distinct from cdot_rest(): cdot is an hgvs *coordinate* data provider, while + AlleleTranslator needs a VRS *sequence* proxy that can derive_refget_accession. + Mirrors dcd_mapping's SeqRepo-backed translator. Uses the same HGVS_SEQREPO_DIR + location as deps.get_seqrepo and the refget router. + """ + return SeqRepoDataProxy(seqrepo()) + + def vrs_mapper(url: Optional[str] = None) -> VRSMap: return VRSMap(DCD_MAP_URL) if not url else VRSMap(url) diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py new file mode 100644 index 000000000..c9b17f4b3 --- /dev/null +++ b/src/mavedb/db/mixins.py @@ -0,0 +1,220 @@ +"""Reusable declarative mixins.""" + +from datetime import datetime +from typing import ClassVar, Optional, Sequence, TypeVar + +from sqlalchemy import ColumnElement, DateTime, Update, func, select, update +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property +from sqlalchemy.orm import Mapped, Session, mapped_column + +T = TypeVar("T", bound="ValidTime") + + +class ValidTime: + """Valid-time versioning for rows that change over time (transaction-time SCD Type 2). + + Applies to either a versioned entity (a row that is replaced by a newer version of the same + logical thing — e.g. a mapping record) or a link/association row (a relationship that comes + and goes). Immutable, content-addressed rows (deduplicated entities like an allele or a + source-versioned annotation record) do not use this — their "what applies now" is answered by + the links that reference them, not by versioning the row itself. + + A row is live while ``valid_to`` is NULL. Superseding a row sets its ``valid_to`` to the + successor's ``valid_from`` instead of deleting it, so the full history is retained and a + point-in-time query is a single half-open ``[valid_from, valid_to)`` predicate. ``current`` + and ``as_of`` express that predicate so call sites never hand-roll it. + + Convention: replacing a live row with a successor goes through :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk), which stamp the retired ``valid_to`` and the new + ``valid_from`` with one timestamp so the handoff is gap-free regardless of transaction + boundaries. ``retire`` (single) / ``retire_live_where`` (bulk) are *withdrawal* primitives — + retiring with no successor; do not pair a bare retire with a separate insert, or a slow job + between the two opens a window where the row is neither live-old nor live-new. The method pairs + mirror: ``retire``/``supersede_with`` act on one row, ``*_live_where`` act on a predicate. + + ``current`` is derived (``valid_to IS NULL``), not stored — one source of truth, no + dual-write to keep consistent. Each table using this mixin should add a partial unique index + over its natural key ``WHERE valid_to IS NULL`` to enforce a single live row per key (the link + key for an association, the logical-entity key for an entity). + + Transaction time only: ``valid_from``/``valid_to`` record when *we* held the row, not when an + external source considered it effective. Tables fed by externally-versioned sources (e.g. a + ClinVar release, a gnomAD version) should add their own source-version column for that axis. + + Consumer contract — a model mixing this in, and any code writing it, MUST honor: + + 1. **Add the partial unique index.** Declare a unique index over the natural key + ``WHERE valid_to IS NULL`` (one live row per key). It is the loud backstop: forgetting to + retire a predecessor before inserting its successor raises ``IntegrityError`` instead of + silently leaving two live rows. + 2. **Replace through supersede, never retire-then-insert.** Use :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk) to swap a live row for a successor. A bare + :meth:`retire` followed by a separate insert is the gap footgun — if a slow job or a commit + falls between them, the retired ``valid_to`` and the new ``valid_from`` come from different + transaction clocks and a point-in-time query lands in a hole where the row is neither + old-live nor new-live. Reserve ``retire``/``retire_live_where`` for genuine withdrawal (no + successor). Gaps only arise on a *same-key* handoff; the cascade in (3) only ever retires + children under a superseded parent's old key, so it never needs a supersede. + 3. **Declare ``__retire_cascade__`` for parent rows with ``ValidTime`` children.** A live link to + a retired parent is stale; retiring a parent must retire its live child links. The ORM + relationship cascade does not do this (it fires only on hard DELETE). Bulk + :meth:`supersede_live_where` refuses to run on a class that declares ``__retire_cascade__`` + precisely because it cannot cascade — supersede such rows one at a time with + :meth:`supersede_with`. + 4. **Filter reads with ``current``/``as_of``.** Never read live rows by assuming a query returns + only current state; constrain with ``current`` (or ``as_of(ts)``) explicitly. Because of (3), + a live link implies a live parent, so allele/parent-side ``current`` queries do not surface + links dangling off retired parents. + """ + + valid_from: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + valid_to: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + + # Names of relationships to child ``ValidTime`` rows that should be retired when this row is. + # A link to a retired row is itself stale, so closing a parent's window must close its live + # links' windows too — the relationship cascade ("all, delete-orphan") only fires on a hard + # DELETE, not on this soft ``valid_to`` close. Each named relationship must target a + # ``ValidTime`` model; ``retire`` issues one bulk UPDATE per relationship. + # + # This covers only the transaction-time axis: "the parent was superseded, so its links are too." + # Links retired on a source-version axis (e.g. a new gnomAD release superseding an + # allele→gnomAD-variant link while the allele itself stays live) are a separate trigger and are + # not driven from here. Cascade is one level deep (bulk UPDATE, not per-row ``retire``); a deeper + # chain would need its own handling. + __retire_cascade__: ClassVar[tuple[str, ...]] = () + + @hybrid_property + def current(self) -> bool: + return self.valid_to is None + + @current.inplace.expression + @classmethod + def _current_expression(cls) -> ColumnElement[bool]: + return cls.valid_to.is_(None) + + @classmethod + def as_of(cls, ts: datetime) -> ColumnElement[bool]: + """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" + return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + + @hybrid_method + def live_at(self, as_of: Optional[datetime]) -> bool: + """Live at ``as_of``, or currently live when ``as_of`` is ``None``. + + The one-call form of the ``as_of(ts) if ts is not None else current`` branch every temporal + read otherwise repeats. Same result as :attr:`current` / :meth:`as_of`, but usable uniformly on + the class, an instance, and an ``aliased()`` class (SQLAlchemy adapts the column refs to it).""" + if as_of is None: + return self.valid_to is None + return self.valid_from <= as_of and (self.valid_to is None or self.valid_to > as_of) + + @live_at.expression + @classmethod + def _live_at_expression(cls, as_of: Optional[datetime]) -> ColumnElement[bool]: + if as_of is None: + return cls.valid_to.is_(None) + return (cls.valid_from <= as_of) & (cls.valid_to.is_(None) | (cls.valid_to > as_of)) + + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: + """Close this row's validity window, cascading to the live child links named in + ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. + + Pass ``at`` to stamp an explicit ``valid_to`` shared with a successor's ``valid_from``; this + is how :meth:`supersede_with` keeps the handoff gap-free. Without ``at`` the close uses + ``func.now()`` (Postgres ``transaction_timestamp``). This is a *withdrawal* primitive on its + own — to replace a row with a successor, use :meth:`supersede_with` so the handoff is closed. + + ``session`` is required only when ``__retire_cascade__`` is non-empty (the cascade issues a + bulk UPDATE per child relationship). A leaf row with no declared cascades retires without one. + The cascade runs even when this row was already closed, so a half-finished prior retire (row + closed, links not) still converges. + """ + stamp = at if at is not None else func.now() + if self.valid_to is None: + self.valid_to = stamp + + if not self.__retire_cascade__: + return + if session is None: + raise ValueError( + f"{type(self).__name__}.retire() needs a session to cascade-retire {self.__retire_cascade__}." + ) + + mapper = sa_inspect(type(self)) + assert mapper is not None # a mapped class always inspects to a Mapper + for name in self.__retire_cascade__: + rel = mapper.relationships[name] + child_cls = rel.mapper.class_ + conditions = [remote == getattr(self, local.key) for local, remote in rel.local_remote_pairs] + session.execute(child_cls.retire_live_where(*conditions, at=at)) + + @classmethod + def retire_live_where(cls, *conditions: ColumnElement[bool], at: Optional[datetime] = None) -> Update: + """An UPDATE that retires the live rows matching ``conditions`` (closes their valid_to). + Only currently-live rows are touched, so already-retired rows keep their original valid_to. + Pass ``at`` to stamp an explicit ``valid_to`` (used by :meth:`supersede`); defaults to + ``func.now()``. + + This is a *withdrawal* primitive — retiring live rows with no successor. To replace live rows + with new ones, use :meth:`supersede_live_where` so retire and insert share one timestamp. + + Usage: ``session.execute(Model.retire_live_where(Model.foo == bar))``. + """ + return ( + update(cls).where(cls.valid_to.is_(None), *conditions).values(valid_to=at if at is not None else func.now()) + ) + + def supersede_with(self: T, session: Session, replacement: T, at: Optional[datetime] = None) -> T: + """Retire this row (cascading to its child links) and insert ``replacement``, stamping the + retired row's ``valid_to`` and the replacement's ``valid_from`` with one timestamp so the + handoff has no gap — independent of how many transactions or how much wall-clock separate + them. The single-row counterpart to :meth:`supersede_live_where`, for superseding one known + row that may carry ``__retire_cascade__`` children. + + Flushes the retire before the insert: the partial unique index (one live row per natural key) + is checked per statement and the unit of work emits INSERTs before UPDATEs, so without the + flush the replacement and the not-yet-closed original would both be live and trip the index. + """ + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + self.retire(session, at=at) + session.flush() + replacement.valid_from = at + session.add(replacement) + return replacement + + @classmethod + def supersede_live_where( + cls, + session: Session, + new_rows: Sequence[T], + *conditions: ColumnElement[bool], + at: Optional[datetime] = None, + ) -> Sequence[T]: + """Retire the live rows matching ``conditions`` and insert ``new_rows``, stamping both sides + with one timestamp so every retired row's ``valid_to`` equals every new row's ``valid_from`` + — a gap-free handoff independent of transaction boundaries. The bulk counterpart to + :meth:`supersede_with` (and the supersede form of :meth:`retire_live_where`), for replacing a + *set* of live rows in one scope (e.g. a score set's derived links) with a freshly computed + set. + + The retire executes immediately (before the inserts flush), so a reused natural key does not + trip the partial unique index. Only for leaf ``ValidTime`` rows: a bulk retire cannot fire + ``__retire_cascade__``, so this refuses to run on a class that declares one — use + :meth:`supersede_with` per row there. + """ + if cls.__retire_cascade__: + raise ValueError( + f"{cls.__name__} declares __retire_cascade__; bulk supersede cannot cascade to children. " + "Use supersede_with per row so child links retire too." + ) + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + session.execute(cls.retire_live_where(*conditions, at=at)) + for row in new_rows: + row.valid_from = at + session.add_all(new_rows) + return new_rows diff --git a/src/mavedb/deps.py b/src/mavedb/deps.py index 8e4ce5106..712a6e5a3 100644 --- a/src/mavedb/deps.py +++ b/src/mavedb/deps.py @@ -1,4 +1,3 @@ -import os from typing import Any, AsyncGenerator, Generator from arq import ArqRedis, create_pool @@ -6,7 +5,7 @@ from cdot.hgvs.dataproviders import RESTDataProvider from sqlalchemy.orm import Session -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo from mavedb.db.session import SessionLocal from mavedb.worker.settings import RedisWorkerSettings @@ -33,5 +32,4 @@ def hgvs_data_provider() -> RESTDataProvider: def get_seqrepo() -> SeqRepo: - seqrepo_dir = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") - return SeqRepo(seqrepo_dir) + return seqrepo() diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py new file mode 100644 index 000000000..4e161dbf5 --- /dev/null +++ b/src/mavedb/lib/allele_annotations.py @@ -0,0 +1,151 @@ +"""Digest-keyed annotation assembly over the deduplicated allele model. + +Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar +assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride +annotations *alongside* the spec-pure Cat-VRS members: Cat-VRS itself carries no annotation slot, so +annotations travel beside it in a flat digest-keyed map. + +All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on +``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past +instant via ``live_at``, defaulting to the currently-live rows. Absence is normal and encoded as +``None``/empty — annotations are sparse and computed at all levels with no fixed level→source +mapping (D44/D44b): a genomic allele *may* carry VEP, a coding allele *may* match gnomAD. + +VEP and gnomAD are one-live-per-allele, so each is at most one value; ClinVar is multi-live (one +live link per release), so it is a list. + +This is the shared assembler for both ``GET /variants/{urn}`` (all of a variant's alleles) and +``GET /alleles/{digest}`` (a single allele) — pass whichever allele set the caller has in hand. +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, Sequence + +from sqlalchemy import Result, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class VepAnnotation: + """A VEP most-severe functional consequence and the Ensembl release it was resolved under.""" + + consequence: Optional[str] + source_version: str + + +@dataclass(frozen=True) +class GnomadAnnotation: + """gnomAD population frequency for an allele, from its live gnomAD link.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] + db_version: str + db_identifier: str + + +@dataclass(frozen=True) +class ClinvarAnnotation: + """One ClinVar assertion for an allele (one per live release — an allele may carry several).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] + clinvar_allele_id: str + db_version: str + + +@dataclass +class AlleleAnnotations: + """The external annotations for a single allele. Every field is sparse — a source with no data + for the allele is ``None`` (vep/gnomad) or an empty list (clinvar).""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = field(default_factory=list) + + +def get_allele_annotations( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleAnnotations]: + """Assemble the digest-keyed annotation map for ``alleles``. + + Returns one :class:`AlleleAnnotations` per distinct ``vrs_digest`` present in ``alleles``. An + allele with no annotations still gets an (empty) entry, so the map's keys mirror the alleles + handed in — the envelope can join every Cat-VRS member to a (possibly empty) annotation block. + ``as_of`` reconstructs each source at a past instant; it defaults to the currently-live rows. + """ + # id -> digest for the alleles we were handed; digest is the map key the envelope joins on. + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + annotations: dict[str, AlleleAnnotations] = { + digest: AlleleAnnotations() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return annotations + + allele_ids = list(digest_by_id.keys()) + + # VEP — at most one live consequence per allele. + vep_rows = db.execute( + select( + VepAlleleConsequence.allele_id, + VepAlleleConsequence.functional_consequence, + VepAlleleConsequence.source_version, + ) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.live_at(as_of)) + ).tuples() + for allele_id, consequence, source_version in vep_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].vep = VepAnnotation(consequence=consequence, source_version=source_version) + + # gnomAD — at most one live link per allele, resolving to a frequency record. + gnomad_rows: Result[tuple[int, GnomADVariant]] = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ) + for allele_id, gnomad_variant in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].gnomad = GnomadAnnotation( + allele_frequency=gnomad_variant.allele_frequency, + allele_count=gnomad_variant.allele_count, + allele_number=gnomad_variant.allele_number, + faf95_max=gnomad_variant.faf95_max, + db_version=gnomad_variant.db_version, + db_identifier=gnomad_variant.db_identifier, + ) + + # ClinVar — multi-live (one live link per release), so each allele accumulates a list. + clinvar_rows: Result[tuple[int, ClinvarControl]] = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ) + for allele_id, clinvar_control in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].clinvar.append( + ClinvarAnnotation( + clinical_significance=clinvar_control.clinical_significance, + clinical_review_status=clinvar_control.clinical_review_status, + clinvar_variation_id=clinvar_control.clinvar_variation_id, + clinvar_allele_id=clinvar_control.db_identifier, + db_version=clinvar_control.db_version, + ) + ) + + return annotations diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py new file mode 100644 index 000000000..109703ede --- /dev/null +++ b/src/mavedb/lib/allele_detail.py @@ -0,0 +1,200 @@ +"""The allele-detail view backing ``GET /alleles/{digest|CAID}``. + +The allele-grain counterpart of :mod:`lib.variant_detail`. Where the variant view anchors on a +*measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) +and serves that allele's identity, its full cross-layer **equivalence class**, and the digest-keyed +annotations that ride alongside. + +It is measurement-agnostic, and that shapes what it does *not* carry: + +- No score, counts, classification, or version standing. Those are properties of a measured variant, + not of an allele (an allele is deduplicated by digest and shared across every score set mapping to it). +- No spec-pure Cat-VRS. The ``CategoricalVariant`` / ``DefiningAlleleConstraint`` is emitted only by the + measurement view, rooted at the measured allele. Here the members are labelled on a flat map + *relative to the focus allele*. + +**The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): +every allele co-linked to any live record touching the focus. That is the discoverable "everything +related to this change" set. Each member is then labelled relative to the focus allele. Even though this +level is unable to determine which allele was actually measured, we can still say how a given allele relates +to the focus allele (e.g., faithful, candidate, convergent) based on the graph structure. + +``as_of`` reconstructs the molecular layer, class membership + annotations, at the past instant. The +focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to this instant. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity +from mavedb.lib.alleles import get_allele_translations +from mavedb.lib.cat_vrs import CatVrsRelation +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +@dataclass(frozen=True) +class AlleleDetail: + """The assembled allele-detail envelope (transit; serialized by ``view_models.allele_detail``).""" + + # Own identity (the focus allele itself). + digest: str + level: Optional[str] + """The sequence level of the allele (e.g., nucleotide or amino acid).""" + hgvs: Optional[str] + """The reference-frame HGVS string (exactly one of hgvs_g/c/p, depending on level).""" + clingen_allele_id: Optional[str] + vrs: Optional[dict[str, Any]] + + # Cross-layer equivalence class (all alleles equivalent to the focus). + alleles: dict[str, AlleleIdentity] + """The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. + Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry + `is_focus=True`.""" + + # External annotations. + annotations: dict[str, AlleleAnnotations] + """External annotations for each member of the cross-layer equivalence class, keyed by VRS digest.""" + + +def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: Optional[datetime]) -> dict[str, str]: + """A symmetric ``digest -> digest`` map pairing each focus allele with its c↔g projection. + + A ``projection_group`` is one c↔g pair, local to a single mapping record. The projection is the ≤1 + *other* allele sharing the same ``projection_group`` as the focus allele. Reading any one live link + is safe: the focus holds one live link per record it appears in, and each record re-encodes the same + change onto the same content-addressed digests, so the pairing is record-independent (assuming those + records agree on genome assembly). + + This is the one place the projection-vs-convergent-encoding distinction needs the link-level + ``projection_group`` — every other member is classified by level alone. + """ + pairs: dict[str, str] = {} + for focus in focus_alleles: + if focus.vrs_digest is None: + continue + + # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. + link = db.scalar( + select(MappingRecordAllele) + .where(MappingRecordAllele.allele_id == focus.id) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.live_at(as_of)) + .limit(1) + ) + if link is None: + continue + + projection_digests = db.scalars( + select(Allele.vrs_digest) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecordAllele.mapping_record_id == link.mapping_record_id) + .where(MappingRecordAllele.projection_group == link.projection_group) + .where(MappingRecordAllele.allele_id != focus.id) + .where(MappingRecordAllele.live_at(as_of)) + ).all() + for projection in projection_digests: + if projection is not None: + pairs[focus.vrs_digest] = projection + pairs.setdefault(projection, focus.vrs_digest) + + return pairs + + +def _member_label( + *, focus_is_protein: bool, member_level: Optional[str], is_projection: bool +) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: + """The (relation, derivation) for a non-focus member, read *relative to the focus*. + + - **protein focus** → every nt member is a reverse-translation ``candidate`` that ``encodes`` it. + - **nt focus** → The protein consequence is ``translation_of`` (a deterministic ``projection`` of the focus). + The focus's projection partner is ``coordinate_representation_of`` (also a deterministic ``projection``). + Any other nt is a convergent encoding: ``co_encodes`` and ``convergent`` (a distinct change which shares the + ultimate protein consequence of the focus). + """ + if focus_is_protein: + if member_level in NUCLEOTIDE_LEVELS: + return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + + return None, None + + if member_level == SequenceLevel.protein.value: + return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION + if member_level in NUCLEOTIDE_LEVELS: + if is_projection: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT + + return None, None + + +def get_allele_detail( + db: Session, focus: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None +) -> AlleleDetail: + """Assemble the allele-detail envelope anchored on ``focus``. + + ``focus_digests`` is the set of VRS digests that are the *focus* of this view. A single digest for + a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The + equivalence class is every allele co-linked to a live record touching the ``focus`` + (:func:`get_allele_translations`). An orphan allele falls back to ``focus`` alone. Non-focus members + are labelled by :func:`_member_label`. + """ + equivalence_class = get_allele_translations(db, focus.id, as_of=as_of) or [focus] + annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) + + focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [focus] + focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) + + pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of) + # The focus's projection(s) that are not themselves focus (for a CAID fetch the projection pair is + # jointly the focus, so this is empty and they are simply flagged is_focus instead). + projection_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests + + alleles: dict[str, AlleleIdentity] = {} + for member in equivalence_class: + digest = member.vrs_digest + if digest is None: + continue + + # Determine the relation and derivation for non-focus members. Focus + # members have no relation or derivation relative to themselves. + relation, derivation = None, None + if digest not in focus_digests: + rel, der = _member_label( + focus_is_protein=focus_is_protein, + member_level=member.level, + is_projection=digest in projection_digests, + ) + relation = rel.value if rel is not None else None + derivation = der + + alleles[digest] = AlleleIdentity( + level=member.level, + hgvs=member.hgvs_g or member.hgvs_c or member.hgvs_p, + clingen_allele_id=member.clingen_allele_id, + is_focus=digest in focus_digests, + relation=relation, + derivation=derivation, + projection_of=pairs.get(digest), + ) + + return AlleleDetail( + digest=focus.vrs_digest or "", + level=focus.level, + hgvs=focus.hgvs_g or focus.hgvs_c or focus.hgvs_p, + clingen_allele_id=focus.clingen_allele_id, + vrs=vrs_object_from_mapped_variant(focus.post_mapped).model_dump(mode="json", exclude_none=True) + if focus.post_mapped is not None + else None, + alleles=alleles, + annotations=annotations, + ) diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py new file mode 100644 index 000000000..1b065b223 --- /dev/null +++ b/src/mavedb/lib/allele_identity.py @@ -0,0 +1,96 @@ +"""The MaveDB molecular-identity view of an allele — shared by variant detail and allele detail. + +Both serving views present a set of alleles keyed by VRS digest, each labelled by how it relates to +the allele the view is anchored on (its *focus*). The two views differ only in what the focus is: +``GET /variants/{urn}`` focuses the **measured** allele, ``GET /alleles/{digest}`` focuses the +**queried** allele. The identity shape is shared and its labels are always read *relative to the +focus*. + +This module owns that shared shape (:class:`AlleleIdentity`) and its provenance vocabulary +(:class:`AlleleDerivation`). The Cat-VRS structural relation codes live in :mod:`lib.cat_vrs` +(``CatVrsRelation``); this module carries the derivation (confidence/provenance) axis. + +Why ``relation`` and ``derivation`` are separate axes +---------------------------------------------------- +The two axes describe the *same* pair of alleles — member and focus — but ask questions from different +domains. ``relation`` is **biological**: how does this allele relate to the focus? ``derivation`` is +**provenance**: how did we arrive at this allele from the focus, and hence how much should it be +trusted? Neither answer determines the other, because a biological relationship does not say how it was +found and a route does not say what it found. + +Half of that independence is already visible in the tree: ``projection`` covers *both* ``translation_of`` +(nt→protein) and ``coordinate_representation_of`` (nt↔nt) — one route, two different biological +relationships. The other half is latent only because today exactly one pipeline path produces each +relationship. A ``co_encodes`` allele is biologically "a different codon encoding the same protein +change" however we reached it; reverse translation from the focus, a registry walk, and an independent +measurement in another score set are three different provenance claims about the same biology. + +Two consequences that make the split load-bearing rather than tidy: + +1. **Cat-VRS cannot express confidence, by design.** ``encodes`` is a claim about molecules: this + nucleotide allele encodes that protein change. It is equally true of a codon we measured and one + we guessed. A structural representation spec has no vocabulary for "we do not know which of these + six codons was in the well". +2. **``candidate`` is a fact about our data, not about the molecule.** A protein-level assay reported + an amino-acid change; which codon produced it is unknown *to MaveDB*, not unknown biologically. + Attempting to encode that on a molecular-relationship axis would misstate what the molecule does. + +Today's one-to-one mapping between the two vocabularies is **a fact about today's pipeline, not about +these axes. Never infer one from the other.** They are emitted together by +``variant_detail._derivation_for`` and ``allele_detail._member_label``, the single source of truth. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at, *relative to the focus allele*: the + confidence/provenance axis, and the one the UI badges on. + + There is deliberately **no** ``authoritative`` value: the focus allele is marked by + :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful even when a + variant was not explicitly measured. See the module docstring for why this axis is separate from + the Cat-VRS ``relation``, and why neither may be inferred from the other. + """ + + PROJECTION = "projection" + """Deterministic and precise, given (assembly, transcript): the focus's projection + (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein).""" + + CANDIDATE = "candidate" + """Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out + equivalence class, not a precise and deterministic pair.""" + + CONVERGENT = "convergent" + """A distinct, precisely-known nucleotide change that *converges* on the same protein consequence + a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A + separate, unmeasured variant that simply shares the consequence.""" + + +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one allele in a view's ``alleles`` map, keyed by VRS + digest. + + These four axes all read *relative to the view's focus allele*: + + - ``is_focus``: whether this allele is the one the view is anchored on. Its ``relation``/``derivation`` + are ``None``. This is the reference point the others are described against. + - ``relation`` (Cat-VRS, biological): how this allele relates to the focus in a biological sense. + - ``derivation`` (:class:`AlleleDerivation`, provenance): How we arrived at this allele from the focus, + hence how much to trust it. + - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection, when + known. ``None`` for the protein apex, pre-reverse-translation data, or where the pairing is not + resolved in this view. + """ + + is_focus: bool + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + + relation: Optional[str] + derivation: Optional[AlleleDerivation] + projection_of: Optional[str] diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py new file mode 100644 index 000000000..23ca174d6 --- /dev/null +++ b/src/mavedb/lib/allele_measurements.py @@ -0,0 +1,332 @@ +"""The variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id, return every measurement related to it across sequence levels. Two measurements +are related when their mapping records share an allele. A ``CA`` anchors on its nt alleles; a ``PA`` on the +protein allele. Each measurement is labelled by its own measured level relative to the query: ``direct`` +(measured at the query), ``protein_consequence``, or ``nucleotide_encoding``. + +Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the encoding ``c.600T>A`` encode +``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1``. Score set X assays ``c.598T>C`` (nt), score set Y +assays ``p.Phe200Leu`` (protein), score set Z assays ``c.600T>A`` (nt): + + query CA1 → X direct, Y protein_consequence, Z nucleotide_encoding + query PA1 → Y direct, X and Z nucleotide_encoding + +A CA query reaches its protein consequence (Y) through the shared protein node ``PA1``. This is what makes +Y reachable even when a protein assay has not been reverse-translated: its record then links only the +protein node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The +encoding Z is reached without the apex: reverse translation links every record to its full synonymous nt set, +so Z's record already links ``CA1`` directly. + +The protein-consequence step is resolved once, in :func:`_resolve_protein_apex`, which also measures it: +a consequence resolving to more than one protein id (a data-integrity signal) is logged, and the number of +consequences reached with no reverse-translation data of their own is written to the request log. + +Authorization (``has_permission``, per ``lib/score_sets.py``): score-set READ gates whether a measurement +appears; calibration READ gates only its inline classification. ``as_of`` reconstructs which records and +links were live at a past instant. + +Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the record-scoped detail of one +measurement. +""" + +import logging +from dataclasses import dataclass +from datetime import date, datetime +from enum import Enum +from typing import Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, aliased, joinedload + +from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength +from mavedb.lib.types.authentication import UserData +from mavedb.lib.variants import variant_score +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +class MeasurementRelationship(str, Enum): + """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" + + direct = "direct" # assayed at the queried allele + protein_consequence = "protein_consequence" # protein measurement of a CA's consequence + nucleotide_encoding = "nucleotide_encoding" # an nt measurement of the protein apex's encoding + + +@dataclass(frozen=True) +class AlleleMeasurement: + """One measurement in the query's equivalence class. + + ``assay_level`` is the sequence level at which this measurement was assayed. ``preferred_classification`` + is the readable classification the UI will default to. + """ + + variant_urn: str + score: Optional[float] + assay_level: Optional[SequenceLevel] + relationship: str + assay_level_hgvs: Optional[str] + submitted_hgvs: Optional[str] + score_set_urn: str + score_set_title: str + preferred_classification: Optional[ScoreCalibrationFunctionalClassification] + is_current: bool + superseded_by_score_set: Optional[str] + + +@dataclass(frozen=True) +class _ProteinApex: + """Protein consequence(s) co-membered with a CA anchor's nt alleles — the shared node a CA query + reaches protein measurements through. One PAID is well-formed; ``len(caids) > 1`` is a data-quality + signal (a duplicate or divergent consequence). ``unregistered`` counts apex rows without a PAID yet. + + nt→protein is deterministic only per (assembly, transcript), and the anchor includes the CAID's + transcript-agnostic genomic allele. So records targeting different transcripts of one gene reach + different consequences legitimately, and >1 PAID is a signal to investigate rather than proof of + corruption. + """ + + allele_ids: list[int] + caids: list[str] + unregistered: int + + +def _preferred_classification( + db: Session, variant: Variant, *, user_data: Optional[UserData] +) -> Optional[ScoreCalibrationFunctionalClassification]: + """The variant's preferred *readable* functional classification, or ``None``. + + Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence + within a tier, then ``id`` for determinism. RUO calibrations are excluded outright, and calibrations + the caller can't read are skipped rather than blanking a readable call. + """ + candidates = [ + (classification, calibration) + for classification, calibration in db.execute( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + .where(ScoreCalibration.research_use_only.is_(False)) + ).all() + if has_permission(user_data, calibration, Action.READ).permitted + ] + if not candidates: + return None + + def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]) -> tuple: + classification, calibration = pair + magnitude, _ = classification_evidence_strength(classification) + return ( + *calibration_preference_key(calibration), + -magnitude, + calibration.id, + ) + + return min(candidates, key=preference)[0] + + +def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: + """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) + first; newest-published first; then URN for a stable tiebreak.""" + magnitude, direction = classification_evidence_strength(measurement.preferred_classification) + return ( + 0 if measurement.is_current else 1, + 0 if measurement.relationship == MeasurementRelationship.direct else 1, + 0 if measurement.preferred_classification is not None else 1, + -magnitude, + direction, + -(published_date.toordinal()) if published_date is not None else 0, + measurement.variant_urn, + ) + + +def _resolve_protein_apex(db: Session, anchor_ids: list[int], *, as_of: Optional[datetime]) -> _ProteinApex: + """The protein consequence(s) co-membered with the nt ``anchor_ids`` via a shared live record. + + The single site the apex is resolved, so its cardinality (see :class:`_ProteinApex`) is measured + rather than being a silent side effect of the union. Reads ``clingen_allele_id`` to count PAIDs. + """ + anchor_link = aliased(MappingRecordAllele) + rows = db.execute( + select(Allele.id, Allele.clingen_allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == SequenceLevel.protein.value) + .distinct() + ).all() + + return _ProteinApex( + allele_ids=[row.id for row in rows], + caids=sorted({row.clingen_allele_id for row in rows if row.clingen_allele_id is not None}), + unregistered=sum(1 for row in rows if row.clingen_allele_id is None), + ) + + +def get_allele_measurements( + db: Session, + clingen_allele_id: str, + *, + user_data: Optional[UserData], + include_superseded: bool = False, + as_of: Optional[datetime] = None, +) -> list[AlleleMeasurement]: + """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live + record. + + A CA query returns ``direct`` measurements, its ``protein_consequence``, and the ``nucleotide_encoding`` + members. A PA query returns its ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. + """ + anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() + if not anchor: + return [] + + anchor_ids = [row.id for row in anchor] + entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) + + # A CA query always includes its protein consequence, reached through the shared protein node, so it is + # reachable even when a protein assay isn't reverse-translated yet. This is calculated transitively, but + # the calculation should be deterministic. + apex: Optional[_ProteinApex] = None + if not entry_is_protein: + apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of) + anchor_ids += apex.allele_ids + + # Warn if multiple distinct protein apexes were resolved for this CA. + if len(apex.caids) > 1: + logger.warning( + msg=( + f"ClinGen allele {clingen_allele_id} resolved to {len(apex.caids)} distinct protein " + f"apexes ({', '.join(apex.caids)}); measurements may conflate consequences." + ), + extra=logging_context(), + ) + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id.in_(anchor_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + if not record_ids: + return [] + + # Records that carry a derived (RT) link of their own. A protein consequence whose record has none was + # reached only through the apex of an already resolved protein consequence. + records_with_derived_links: set[int] = ( + set( + db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.is_authoritative.is_(False)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) + if apex is not None + else set() + ) + + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. + authoritative = aliased(MappingRecordAllele) + rows = ( + db.execute( + select(MappingRecord, Variant, Allele) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join( + authoritative, + and_( + authoritative.mapping_record_id == MappingRecord.id, + authoritative.is_authoritative.is_(True), + authoritative.live_at(as_of), + ), + ) + .join(Allele, Allele.id == authoritative.allele_id) + .where(MappingRecord.id.in_(record_ids)) + .where(MappingRecord.live_at(as_of)) + .options(joinedload(Variant.score_set)) + ) + .tuples() + .all() + ) + + measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + protein_consequence_count = 0 + apex_only_unresolved_count = 0 + for record, variant, measured_allele in rows: + score_set = variant.score_set + if not has_permission(user_data, score_set, Action.READ).permitted: + continue + + # Don't leak an unreadable superseding score set. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + is_current = superseding is None + if not is_current and not include_superseded: + continue + + # Label by the measured (authoritative) allele's level relative to the query. + if measured_allele.clingen_allele_id == clingen_allele_id: + relationship = MeasurementRelationship.direct + elif measured_allele.level == SequenceLevel.protein.value: + relationship = MeasurementRelationship.protein_consequence + else: + relationship = MeasurementRelationship.nucleotide_encoding + + # Apex-only provenance: a protein consequence whose record has no derived link was reached purely + # through the apex. + if apex is not None and relationship == MeasurementRelationship.protein_consequence: + protein_consequence_count += 1 + if record.id not in records_with_derived_links: + apex_only_unresolved_count += 1 + + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None + measurement = AlleleMeasurement( + variant_urn=variant.urn or "", + score=variant_score(variant), + assay_level=assay_level, + relationship=relationship, + assay_level_hgvs=record.hgvs_assay_level, + submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, + score_set_urn=score_set.urn or "", + score_set_title=score_set.title or "", + preferred_classification=_preferred_classification(db, variant, user_data=user_data), + is_current=is_current, + superseded_by_score_set=superseding.urn if superseding is not None else None, + ) + measurements.append((measurement, score_set.published_date)) + + # Publish the apex provenance to the request log so the ambiguous-apex and apex-only-unresolved rates are observable. + if apex is not None: + save_to_logging_context( + { + "apex_paid_count": len(apex.caids), + "apex_unregistered_count": apex.unregistered, + "measurements_protein_consequence": protein_consequence_count, + "measurements_apex_only_unresolved": apex_only_unresolved_count, + } + ) + + measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) + return [measurement for measurement, _ in measurements] diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py new file mode 100644 index 000000000..08f13ef0b --- /dev/null +++ b/src/mavedb/lib/alleles.py @@ -0,0 +1,116 @@ +"""Allele-graph queries over the deduplicated allele model. + +These traverse the ``MappingRecordAllele`` link graph rather than any external identifier, because an +allele's cross-layer equivalence (its genomic / coding / protein representations) is established by the +mapping + reverse-translation process and materialized as co-membership in a ``MappingRecord``'s allele +set. No single identifier spans the layers: ClinGen's canonical allele id (CAID) covers only the +nucleotide layers, the protein allele carries a distinct PA, so the link graph is the only thing that +ties all three together. +""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session, joinedload + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + + +def get_live_record_allele_links( + db: Session, variant_id: int, *, as_of: Optional[datetime] = None +) -> list[MappingRecordAllele]: + """Return the live ``MappingRecordAllele`` links of a variant's single live ``MappingRecord``, + each with its ``allele`` eagerly loaded and its ``is_authoritative`` flag. + + This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within + one variant's own mapping record rather than taking the cross-record union an anchor allele can + belong to. That scope is what the per-variant Cat-VRS transit needs — the variant's measured + (authoritative) allele as the defining representation and exactly its co-linked members, not the + equivalence class assembled from every record that happens to share a deduplicated allele. + + Temporal: defaults to the currently-live record and links (``valid_to IS NULL``). ``as_of`` + applies the same half-open predicate to both the record and the links, so the set is evaluated at + one instant. Returns ``[]`` when the variant has no live record. + """ + record_id = db.scalar( + select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(MappingRecord.live_at(as_of)) + ) + if record_id is None: + return [] + + return list( + db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.mapping_record_id == record_id) + .where(MappingRecordAllele.live_at(as_of)) + .options(joinedload(MappingRecordAllele.allele)) + ).all() + ) + + +def find_variants_by_vrs_identifier( + db: Session, identifier: str, *, as_of: Optional[datetime] = None +) -> list[tuple[Variant, Allele]]: + """Resolve a GA4GH VRS identifier to the variants whose mapping links an allele bearing it. + + Matches ``identifier`` against a deduplicated allele's ``vrs_digest``, then walks ``Allele → + MappingRecordAllele → MappingRecord → Variant``. Because alleles are deduplicated by ``vrs_digest`` + and shared across variants/score sets, one identifier can resolve to several variants; the + paired allele carries the ClinGen id + level the caller surfaces alongside the URN. + + The live link + record set (``valid_to IS NULL`` / ``as_of``) is used to filter results. Returns distinct + ``(variant, matched_allele)`` pairs. Include an ``as_of`` timestamp to reconstruct the set as it stood + at a past instant. + """ + stmt = ( + select(Variant, Allele) + .select_from(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Allele.vrs_digest == identifier) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + ) + + return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] + + +def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: + """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a + ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, + and protein layers. + + The relation is co-membership in a record's allele set, not a shared identifier — see the module + docstring. Because alleles are deduplicated by ``vrs_digest`` and shared across variants/score sets, + the anchor may belong to several ``MappingRecord``s; the result is the union of their allele sets + (normally the same biological equivalence class). Scope by record or score set upstream if a single + context is required. + + Temporal: by default returns the currently-live set (``valid_to IS NULL``). Pass ``as_of`` to + reconstruct the set as it stood at a past instant — both the anchor hop and the fan-out hop apply + the same half-open ``[valid_from, valid_to)`` predicate, so the whole set is evaluated at one + instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, + so filtering the links alone is sufficient. + """ + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id == allele_id) + .where(MappingRecordAllele.live_at(as_of)) + ).all() + if not record_ids: + return [] + + return list( + db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index 83b13b714..44d9847e4 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -1,11 +1,15 @@ """ -This module supports the construction of three main VA-Spec data structures based on the MaveDB MappedVariant object: +This module supports the construction of three main VA-Spec data structures from a variant's +:class:`VariantAnnotationContext` (the new mapping-record substrate — see ``context.py``): - StudyResult See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#study-result-structure - Statement See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#statement-structure - VariantPathogenicityStatement See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 + +Callers build the context at the edge (router/script) via ``variant_annotation_context`` and pass it in; +these builders never touch the database. ``as_of`` selection and supersession are baked into the context. """ from typing import Optional, Sequence, TypeVar, Union @@ -13,28 +17,30 @@ from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement +from mavedb.lib.annotation.calibration import ( + calibration_scope_extension, + calibrations_available_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) from mavedb.lib.annotation.classification import functional_classification_of_variant +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.annotation.eligibility import ( + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, - mapped_variant_to_pathogenicity_statement, -) -from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result -from mavedb.lib.annotation.util import ( - calibration_scope_extension, - calibrations_available_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, + functional_statement, + pathogenicity_statement, ) +from mavedb.lib.annotation.study_result import variant_impact_study_result from mavedb.lib.permissions.principal import Principal -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration Annotation = TypeVar( @@ -56,90 +62,86 @@ def _disclosing_calibration_scope(annotation: Annotation, calibrations: Sequence ) -def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: +def variant_study_result(context: VariantAnnotationContext) -> ExperimentalVariantFunctionalImpactStudyResult: # A study result reports the measured score and carries no calibration-derived evidence, so its scope # is public regardless of viewer. Disclosed anyway, so that a missing scope never has to be read as # "public" or "generated before disclosure existed". - return _disclosing_calibration_scope(mapped_variant_to_experimental_variant_impact_study_result(mapped_variant), []) + return _disclosing_calibration_scope(variant_impact_study_result(context), []) def variant_functional_impact_statement( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False, principal: Optional[Principal] = None, ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal + context, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) eligible_calibrations = calibrations_available_for_annotation( - mapped_variant, + context, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal, ) # Select the calibration with the strongest evidence - strongest_calibration, strongest_range = select_strongest_functional_calibration( - mapped_variant, eligible_calibrations - ) + strongest_calibration, strongest_range = select_strongest_functional_calibration(context, eligible_calibrations) if not strongest_calibration: return None # Get the classification from the strongest range # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Build evidence lines for all eligible calibrations functional_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) + functional_evidence.append(functional_evidence_line(context, score_calibration, [study_result])) return _disclosing_calibration_scope( - mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + functional_statement( + context, functional_proposition, functional_evidence, strongest_calibration, classification ), eligible_calibrations, ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False, principal: Optional[Principal] = None, ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal + context, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + clinical_proposition = variant_pathogenicity_proposition(context) eligible_calibrations = calibrations_available_for_annotation( - mapped_variant, + context, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal, ) # Select the calibration with the strongest evidence - strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( - mapped_variant, eligible_calibrations - ) + strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration(context, eligible_calibrations) if not strongest_calibration: return None # Get the classification from the strongest range (used for the functional statement within clinical evidence) # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Note: strongest_range is used in the pathogenicity statement for ACMG classification # If None, the statement will use UNCERTAIN_SIGNIFICANCE @@ -147,41 +149,44 @@ def variant_pathogenicity_statement( # Build evidence lines for all eligible calibrations clinical_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, [functional_evidence], score_calibration, classification + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + functional_impact_statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, classification ) clinical_evidence.append( - acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + acmg_evidence_line(context, score_calibration, clinical_proposition, [functional_impact_statement]) ) return _disclosing_calibration_scope( - mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + pathogenicity_statement( + context, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range ), eligible_calibrations, ) def variant_highest_level_annotation( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, principal: Optional[Principal] = None, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ - Build the single highest-materialized VA-Spec layer for a mapped variant. + Build the single highest-materialized VA-Spec layer for a variant's annotation context. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. - Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + + Returns None when the variant lacks the target/mapping data required to build any layer. The viewer decides which layer is reachable as well as what the layer contains: a variant whose only calibration is invisible to this principal degrades to a study result rather than yielding a statement with nothing in it. """ + try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant, principal=principal): - return variant_pathogenicity_statement(mapped_variant, principal=principal) - if can_annotate_variant_for_functional_statement(mapped_variant, principal=principal): - return variant_functional_impact_statement(mapped_variant, principal=principal) - return variant_study_result(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(context, principal=principal): + return variant_pathogenicity_statement(context, principal=principal) + if can_annotate_variant_for_functional_statement(context, principal=principal): + return variant_functional_impact_statement(context, principal=principal) + return variant_study_result(context) + except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/calibration.py b/src/mavedb/lib/annotation/calibration.py new file mode 100644 index 000000000..13565a84f --- /dev/null +++ b/src/mavedb/lib/annotation/calibration.py @@ -0,0 +1,259 @@ +"""Score-calibration eligibility and selection for VA-Spec annotation. + +Which of a score set's calibrations may back an annotation (``score_calibration_may_be_used_for_annotation``) +and, among the eligible ones, which carries the strongest evidence for a given variant +(``select_strongest_*``). The selection resolves ties/conflicts to the conservative call (normal / uncertain +significance). Consumed by the annotation builders in ``annotate.py``. +""" + +from typing import Iterable, Literal, Optional + +from ga4gh.core.models import Extension +from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided + +from mavedb.lib.annotation.classification import ( + ExperimentalVariantFunctionalImpactClassification, + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification + +CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" +"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" + + +def calibrations_available_for_annotation( + context: VariantAnnotationContext, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, +) -> list[ScoreCalibration]: + """ + Select the calibrations on a mapped variant's score set that may build the requested annotation. + + Two independent questions decide this, and they are asked by different collaborators. + - Eligibility: Does this calibration carry the classifications the annotation type needs, and is + its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. + - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule + is stricter than its score set's. + + Args: + context (VariantAnnotationContext): The context containing the variant whose score set's calibrations are considered. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller, so a function that omits it gets public calibrations only. + + Returns: + list[ScoreCalibration]: The eligible, visible calibrations, in score set order. + """ + viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) + + return [ + score_calibration + for score_calibration in viewer.visible(context.variant.score_set.score_calibrations) + if score_calibration_may_be_used_for_annotation( + score_calibration, + annotation_type, + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + ) + ] + + +def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: + """ + Describe the principal an annotation was built for, given the calibrations behind it. + + VA-Spec statements carry no stable identifier, so two callers can receive materially different + statements from the same URL. Naming the scope on the object itself is what keeps that honest: a + consumer holding a record can tell whether it is the one anyone would get, or one widened by the + requester's own access. + + Args: + calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. + + Returns: + Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing + calibration is private and ``public`` otherwise. + """ + if any(calibration.private for calibration in calibrations): + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="restricted", + description=( + "Built from at least one private score calibration, visible to the requesting viewer. " + "Another viewer requesting this variant may receive fewer evidence lines, or none." + ), + ) + + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="public", + description=( + "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." + ), + ) + + +def score_calibration_may_be_used_for_annotation( + score_calibration: ScoreCalibration, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a score calibration may be used for annotation based on its properties. + + This function evaluates whether a given score calibration is suitable for use in + annotation by checking its research use only status and the presence of required + classifications based on the annotation type. + + Args: + score_calibration (ScoreCalibration): The score calibration to evaluate. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation + being considered, which determines the required classifications for validity. + allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations + marked as research use only for annotation. Defaults to False. + + Returns: + bool: True if the score calibration can be used for annotation, False otherwise. + """ + if score_calibration.research_use_only and not allow_research_use_only_calibrations: + return False + + if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: + return False + + if annotation_type == "pathogenicity" and all( + fr.acmg_classification is None for fr in score_calibration.functional_classifications + ): + return False + + return True + + +def select_strongest_functional_calibration( + context: VariantAnnotationContext, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for functional classification. + + In case of ties or conflicting classifications, defaults to normal classification. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should be classified as INDETERMINATE but still receive annotations. + """ + if not calibrations: + return None, None + + # Collect all calibrations and their classifications + candidates: list[ + tuple[ + ScoreCalibration, + ScoreCalibrationFunctionalClassification, + ExperimentalVariantFunctionalImpactClassification, + ] + ] = [] + + for calibration in calibrations: + functional_range, classification = functional_classification_of_variant(context.variant, calibration) + if functional_range is not None: + candidates.append((calibration, functional_range, classification)) + + # If variant is not in any range, return first calibration with None to indicate INDETERMINATE + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Check if all classifications agree + classifications = [c[2] for c in candidates] + if all(cls == classifications[0] for cls in classifications): + # All agree, return the first one + return candidates[0][0], candidates[0][1] + + # Conflict exists: default to normal classification + normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] + if normal_candidates: + return normal_candidates[0][0], normal_candidates[0][1] + + # If no normal classification, return the first candidate + return candidates[0][0], candidates[0][1] + + +def select_strongest_pathogenicity_calibration( + context: VariantAnnotationContext, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for pathogenicity classification. + + Uses ACMG evidence strength to determine the strongest evidence. + In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain + significance by returning None for the functional range. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should receive annotations even though it's not classified in any range. + """ + if not calibrations: + return None, None + + # Define evidence strength ordering (higher index = stronger evidence) + # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. + # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. + strength_order = { + None: 0, + VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, + VaSpecStrengthOfEvidenceProvided.MODERATE: 2, + VaSpecStrengthOfEvidenceProvided.STRONG: 3, + VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, + } + + # Collect all calibrations with their evidence strength and criterion + candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] + + for calibration in calibrations: + functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( + context.variant, calibration + ) + if functional_range is not None: + strength_value = strength_order.get(evidence_strength, 0) + is_benign = criterion.name.startswith("B") if criterion else False + candidates.append((calibration, functional_range, strength_value, is_benign)) + + # If variant is not in any range, return first calibration with None + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Find the maximum strength + max_strength = max(c[2] for c in candidates) + strongest_candidates = [c for c in candidates if c[2] == max_strength] + + # If only one with max strength, return it + if len(strongest_candidates) == 1: + return strongest_candidates[0][0], strongest_candidates[0][1] + + # Tie: check for conflicting evidence (both benign and pathogenic) + has_benign = any(c[3] for c in strongest_candidates) + has_pathogenic = any(not c[3] for c in strongest_candidates) + + # If there's a conflict between benign and pathogenic evidence of equal strength, + # return None for the functional range to indicate uncertain significance + if has_benign and has_pathogenic: + return strongest_candidates[0][0], None + + # Otherwise return the first of the strongest + return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index c7707eb1c..b7904637b 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -6,9 +6,9 @@ from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.variant import Variant logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): def _classification_contains_variant( functional_classification: ScoreCalibrationFunctionalClassification, - mapped_variant: MappedVariant, + variant: Variant, containing_classification_ids: Optional[set[int]], ) -> bool: """Whether this classification's score range contains the variant. @@ -33,11 +33,11 @@ def _classification_contains_variant( """ if containing_classification_ids is not None: return functional_classification.id in containing_classification_ids - return mapped_variant.variant in functional_classification.variants + return variant in functional_classification.variants def functional_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: @@ -51,20 +51,20 @@ def functional_classification_of_variant( every range. A caller classifying many variants should resolve membership once from the association table; see ``mavedb.lib.csv.variant``. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify functional impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify functional impact." ) for functional_range in score_calibration.functional_classifications: - if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): + if _classification_contains_variant(functional_range, variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -75,7 +75,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ @@ -95,20 +95,20 @@ def pathogenicity_classification_of_variant( Raises ValueError if required calibration, score, or evidence strength is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify clinical impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify clinical impact." ) for pathogenicity_range in score_calibration.functional_classifications: - if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): + if _classification_contains_variant(pathogenicity_range, variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) @@ -147,7 +147,7 @@ def pathogenicity_classification_of_variant( not in VariantPathogenicityEvidenceLine.Criterion._member_names_ ): # pragma: no cover - enforced by model validators in FunctionalClassification view model raise ValueError( - f"Variant {mapped_variant.variant.urn} is contained in a clinical calibration range with an invalid criterion." + f"Variant {variant.urn} is contained in a clinical calibration range with an invalid criterion." " Unable to classify clinical impact." ) diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py new file mode 100644 index 000000000..bf1e3d13b --- /dev/null +++ b/src/mavedb/lib/annotation/context.py @@ -0,0 +1,80 @@ +"""Allele-graph variant context for annotation builders. + +Serves ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate to VA-Spec builders, +and provides a single point of truth for the live (or as-of) mapping record, authoritative allele, +and pre-built VA proposition subject. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.variant import Variant + + +@dataclass +class VariantAnnotationContext: + """A variant's annotation inputs, sourced from it's allele-graph. + + ``record`` is the live (or as-of) ``MappingRecord``. Its ``mapping_api_version`` / ``mapped_date`` + supply VA provenance, and its ``ValidTime`` is what ``as_of`` and supersession are evaluated against. + ``measured_allele`` is the authoritative allele (the assayed representation) and its ``post_mapped`` VRS is + the concrete study-result focus. ``subject_variant`` is the VA *proposition* subject. When projections + exist, this is a Cat-VRS ``CategoricalVariant`` object. Otherwise, it is the measured ``MolecularVariation``. + """ + + variant: Variant + record: MappingRecord + measured_allele: Allele + subject_variant: MolecularVariation | CategoricalVariant + as_of: Optional[datetime] + + +def variant_annotation_context( + db: Session, variant: Variant, *, as_of: Optional[datetime] = None +) -> Optional[VariantAnnotationContext]: + """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. + + Returns ``None`` when the variant is unmapped at ``as_of`` or an authoritative allele that carries + no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link fetch; the Cat-VRS + transit is built from the same links, so the subject costs no redundant query. + """ + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + if record is None: + return None + + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + measured_allele = next((link.allele for link in links if link.is_authoritative), None) + if measured_allele is None or measured_allele.post_mapped is None: + return None + + # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a + # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* + # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member + # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured + # allele. + transit = build_categorical_variant(links, name=variant.urn or "", include_convergent=False) + if transit is not None and len(transit.categorical_variant.members) > 1: + subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant + else: + subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) + + return VariantAnnotationContext( + variant=variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant, + as_of=as_of, + ) diff --git a/src/mavedb/lib/annotation/contribution.py b/src/mavedb/lib/annotation/contribution.py index 4b0de881f..839721b5f 100644 --- a/src/mavedb/lib/annotation/contribution.py +++ b/src/mavedb/lib/annotation/contribution.py @@ -9,8 +9,8 @@ mavedb_user_agent, mavedb_vrs_agent, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.types.annotation import ResourceWithCreationModificationDates -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User @@ -31,17 +31,17 @@ def mavedb_api_contribution() -> Contribution: ) -def mavedb_vrs_contribution(mapped_variant: MappedVariant) -> Contribution: +def mavedb_vrs_contribution(context: VariantAnnotationContext) -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) - object from the provided mapped variant. + object from the variant's live mapping record (the VRS mapper's provenance). """ return Contribution( name="MaveDB VRS Mapper", description="Contribution from the MaveDB VRS mapping software", - # Guaranteed to be a str via DB constraints. - contributor=mavedb_vrs_agent(mapped_variant.mapping_api_version), # type: ignore - date=mapped_variant.mapped_date, # type: ignore + # Both guaranteed non-null via DB constraints on MappingRecord. + contributor=mavedb_vrs_agent(context.record.mapping_api_version), # type: ignore + date=context.record.mapped_date, # type: ignore activityType="human genome sequence mapping process", ) diff --git a/src/mavedb/lib/annotation/document.py b/src/mavedb/lib/annotation/document.py index 039142030..d9e58cfc2 100644 --- a/src/mavedb/lib/annotation/document.py +++ b/src/mavedb/lib/annotation/document.py @@ -7,8 +7,8 @@ from ga4gh.va_spec.base.core import Document from mavedb.constants import MAVEDB_FRONTEND_URL +from mavedb.models.allele import Allele from mavedb.models.experiment import Experiment -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -128,31 +128,34 @@ def score_calibration_as_document(score_calibration: ScoreCalibration) -> Docume ) -def mapped_variant_as_iri(mapped_variant: MappedVariant) -> Optional[IRI]: +def measured_allele_as_iri(allele: Allele) -> Optional[IRI]: """ - Create an IRI as described in for the provided MaveDB mapped variant. Within - the context of VA-Spec, these can be used interchangeably with an equivalent document object for brevity. + Create an IRI as described in for the + measured (authoritative) allele, keyed by its ClinGen allele id. Within the context of VA-Spec, these + can be used interchangeably with an equivalent document object for brevity. ``None`` when the allele + has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None - return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(mapped_variant.clingen_allele_id)}") + return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}") -def mapped_variant_to_document(mapped_variant: MappedVariant) -> Optional[Document]: +def measured_allele_to_document(allele: Allele) -> Optional[Document]: """ Create a [VA Document](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/document.html#document) - object from the provided MaveDB mapped variant. + object from the provided MaveDB measured (authoritative) allele, keyed by its ClinGen allele id. ``None`` + when the allele has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None return Document( - id=mapped_variant.variant.urn, - name="MaveDB Mapped Variant", - documentType="mapped genomic variant description", - # We only reach this point if a IRI is guaranteed to exist - urls=[mapped_variant_as_iri(mapped_variant).root], # type: ignore + id=allele.clingen_allele_id, + name="MaveDB Measured Allele", + documentType="measured allele", + # The measured allele document is keyed by its ClinGen allele id, so the IRI is guaranteed to exist. + urls=[measured_allele_as_iri(allele).root], # type: ignore ) diff --git a/src/mavedb/lib/annotation/eligibility.py b/src/mavedb/lib/annotation/eligibility.py new file mode 100644 index 000000000..e4c029694 --- /dev/null +++ b/src/mavedb/lib/annotation/eligibility.py @@ -0,0 +1,108 @@ +"""Whether a variant is eligible for VA-Spec annotation. + +Gate predicates the annotation builders consult before constructing a statement: a variant needs a real +score (the base assumption) and its score set needs at least one calibration usable for the target +annotation type (:func:`calibration.calibrations_available_for_annotation`). This calibration must be readable +by the requesting principal. Split by annotation type into ``can_annotate_variant_for_pathogenicity_evidence`` / +``can_annotate_variant_for_functional_statement``. +""" + +from typing import Optional + +from mavedb.lib.annotation.calibration import calibrations_available_for_annotation +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.variants import variant_score + + +def _can_annotate_variant_base_assumptions(context: VariantAnnotationContext) -> bool: + """ + Check if a variant meets the basic requirements for annotation. + + This function validates that a variant has the necessary data to proceed with + annotation by checking for a valid score value. + + Args: + context (VariantAnnotationContext): The context containing the variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated (has a non-None numeric score), False otherwise. + """ + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(context.variant) is None: + return False + + return True + + +def can_annotate_variant_for_pathogenicity_evidence( + context: VariantAnnotationContext, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, +) -> bool: + """ + Determine if a variant can be annotated for pathogenicity evidence. + + This function checks if a variant meets all the necessary conditions to receive + pathogenicity evidence annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for pathogenicity evidence annotation. + + Args: + context (VariantAnnotationContext): The context containing the variant to evaluate for pathogenicity evidence annotation eligibility. + + Returns: + bool: True if the variant can be annotated for pathogenicity evidence, False otherwise. + + Notes: + The function performs two main validation checks: + 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. + + Both checks must pass for the variant to be considered eligible for + pathogenicity evidence annotation. + """ + if not _can_annotate_variant_base_assumptions(context): + return False + + return bool( + calibrations_available_for_annotation( + context, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) + + +def can_annotate_variant_for_functional_statement( + context: VariantAnnotationContext, allow_research_use_only_calibrations=False, principal: Optional[Principal] = None +) -> bool: + """ + Determine if a variant can be annotated for functional statements. + + This function checks if a variant meets all the necessary conditions to receive + functional annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for functional annotation. + + Args: + context (VariantAnnotationContext): The context containing the variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated for functional statements, False otherwise. + + Notes: + The function performs two main checks: + 1. Validates base assumptions using _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for functional annotation. + """ + if not _can_annotate_variant_base_assumptions(context): + return False + + return bool( + calibrations_available_for_annotation( + context, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index d30e90542..9ce46b181 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -16,6 +16,7 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -30,18 +31,17 @@ functional_score_calibration_as_method, pathogenicity_score_calibration_as_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration def acmg_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, proposition: VariantPathogenicityProposition, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> VariantPathogenicityEvidenceLine: containing_evidence_range, evidence_outcome, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration + context.variant, score_calibration ) evidence_outcome_code = acmg_evidence_outcome_code( @@ -61,7 +61,7 @@ def acmg_evidence_line( direction_of_evidence = direction_of_support_for_pathogenicity_classification(evidence_outcome) return VariantPathogenicityEvidenceLine( - description=f"Pathogenicity evidence line for {mapped_variant.variant.urn}.", + description=f"Pathogenicity evidence line for {context.variant.urn}.", hasEvidenceItems=list(evidence), specifiedBy=pathogenicity_score_calibration_as_method(score_calibration, evidence_outcome), evidenceOutcome={ @@ -75,7 +75,7 @@ def acmg_evidence_line( directionOfEvidenceProvided=direction_of_evidence, contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], targetProposition=proposition, @@ -91,14 +91,14 @@ def acmg_evidence_line( def functional_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> EvidenceLine: - containing_evidence_range, classification = functional_classification_of_variant(mapped_variant, score_calibration) + containing_evidence_range, classification = functional_classification_of_variant(context.variant, score_calibration) return EvidenceLine( - description=f"Functional evidence line for {mapped_variant.variant.urn}", + description=f"Functional evidence line for {context.variant.urn}", hasEvidenceItems=[StudyResult(root=item) for item in evidence], directionOfEvidenceProvided=direction_of_support_for_functional_classification(classification), evidenceOutcome=MappableConcept( @@ -110,7 +110,7 @@ def functional_evidence_line( specifiedBy=functional_score_calibration_as_method(score_calibration), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], reportedIn=[score_calibration_as_document(score_calibration)], diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py index 04d2506f7..5ffe06377 100644 --- a/src/mavedb/lib/annotation/flatten.py +++ b/src/mavedb/lib/annotation/flatten.py @@ -14,8 +14,8 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant @dataclass(frozen=True) @@ -56,7 +56,7 @@ class FlatAnnotation: def flatten_annotation( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: Optional[ScoreCalibration], containing_classification_ids: Optional[set[int]] = None, ) -> FlatAnnotation: @@ -86,7 +86,7 @@ def flatten_annotation( ) _, functional_classification = functional_classification_of_variant( - mapped_variant, score_calibration, containing_classification_ids + variant, score_calibration, containing_classification_ids ) functional_only = FlatAnnotation( @@ -103,7 +103,7 @@ def flatten_annotation( # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration, containing_classification_ids + variant, score_calibration, containing_classification_ids ) # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 963327c73..143d5d608 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -2,22 +2,77 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition from mavedb.lib.annotation.condition import generic_disease_condition +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.document import experiment_to_document -from mavedb.lib.annotation.util import sequence_feature_for_mapped_variant, variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.types.annotation import SequenceFeature +from mavedb.lib.variants import target_for_variant +from mavedb.models.variant import Variant -def mapped_variant_to_experimental_variant_clinical_impact_proposition( - mapped_variant: MappedVariant, +def sequence_feature_for_variant(variant: Variant) -> SequenceFeature: + """ + Extract the sequence feature (e.g., gene or transcript) associated with a variant. + + This function retrieves the sequence feature from the variant's target data, which is + necessary for generating annotations that reference specific genomic features. Co-located with the + propositions that consume it — its only caller. + + Args: + variant (Variant): The variant whose target gene supplies the sequence feature. + + Returns: + SequenceFeature: Named tuple with: + - `identifier`: sequence feature identifier (gene/transcript ID or name) + - `system`: source/system URL for the identifier + + """ + target = target_for_variant(variant) + if target is None: + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable target gene." + " Unable to extract sequence feature for annotation." + ) + + # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. + # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or + # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. + if target.mapped_hgnc_name: + return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") + + post_mapped_ids = extract_ids_from_post_mapped_metadata( + target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore + ) + if post_mapped_ids: + post_mapped_id = post_mapped_ids[0] + if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): + return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") + elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): + return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") + + return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") + + if target.name: + return SequenceFeature(target.name, "https://www.mavedb.org/") + + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable sequence feature in its target gene data." + " Unable to extract sequence feature for annotation." + ) + + +def variant_pathogenicity_proposition( + context: VariantAnnotationContext, ) -> VariantPathogenicityProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return VariantPathogenicityProposition( - description=f"Variant pathogenicity proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant pathogenicity proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="isCausalFor", objectCondition=generic_disease_condition(), geneContextQualifier=sequence_feature @@ -26,18 +81,18 @@ def mapped_variant_to_experimental_variant_clinical_impact_proposition( ) -def mapped_variant_to_experimental_variant_functional_impact_proposition( - mapped_variant: MappedVariant, +def variant_functional_impact_proposition( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return ExperimentalVariantFunctionalImpactProposition( - description=f"Variant functional impact proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant functional impact proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="impactsFunctionOf", objectSequenceFeature=sequence_feature, - experimentalContextQualifier=experiment_to_document(mapped_variant.variant.score_set.experiment), + experimentalContextQualifier=experiment_to_document(context.variant.score_set.experiment), ) diff --git a/src/mavedb/lib/annotation/statement.py b/src/mavedb/lib/annotation/statement.py index c77d30f1a..7db3b05d8 100644 --- a/src/mavedb/lib/annotation/statement.py +++ b/src/mavedb/lib/annotation/statement.py @@ -10,6 +10,7 @@ ) from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -20,23 +21,22 @@ variant_interpretation_functional_guideline_method, variant_interpretation_pathogenicity_guideline_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -def mapped_variant_to_functional_statement( - mapped_variant: MappedVariant, +def functional_statement( + context: VariantAnnotationContext, proposition: ExperimentalVariantFunctionalImpactProposition, evidence: list[EvidenceLine], score_calibration: ScoreCalibration, functional_classification: ExperimentalVariantFunctionalImpactClassification, ) -> Statement: """ - Create a functional impact statement for a mapped variant. + Create a functional impact statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The functional impact proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -48,11 +48,11 @@ def mapped_variant_to_functional_statement( direction = aggregate_direction_of_evidence(evidence) return Statement( - description=f"Variant functional impact statement for {mapped_variant.variant.urn}.", + description=f"Variant functional impact statement for {context.variant.urn}.", specifiedBy=variant_interpretation_functional_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, @@ -67,18 +67,18 @@ def mapped_variant_to_functional_statement( ) -def mapped_variant_to_pathogenicity_statement( - mapped_variant: MappedVariant, +def pathogenicity_statement( + context: VariantAnnotationContext, proposition: VariantPathogenicityProposition, evidence: list[VariantPathogenicityEvidenceLine], score_calibration: ScoreCalibration, functional_range: Optional[ScoreCalibrationFunctionalClassification], ) -> VariantPathogenicityStatement: """ - Create a pathogenicity statement for a mapped variant. + Create a pathogenicity statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The pathogenicity proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -105,11 +105,11 @@ def mapped_variant_to_pathogenicity_statement( acmg_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE return VariantPathogenicityStatement( - description=f"Variant pathogenicity statement for {mapped_variant.variant.urn}.", + description=f"Variant pathogenicity statement for {context.variant.urn}.", specifiedBy=variant_interpretation_pathogenicity_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index 85dbcbdde..c45457468 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -1,5 +1,6 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -7,30 +8,31 @@ mavedb_vrs_contribution, ) from mavedb.lib.annotation.dataset import score_set_to_data_set -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri from mavedb.lib.annotation.method import ( publication_identifiers_to_method, ) -from mavedb.lib.annotation.util import variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.lib.variants import variant_score -def mapped_variant_to_experimental_variant_impact_study_result( - mapped_variant: MappedVariant, +def variant_impact_study_result( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactStudyResult: + # The study result's focus is the concrete measured allele — never a CategoricalVariant (VA-Spec + # narrows ``focusVariant`` to MolecularVariation). The context guarantees a hydratable post_mapped. return ExperimentalVariantFunctionalImpactStudyResult( - description=f"Variant effect study result for {mapped_variant.variant.urn}.", - focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=mapped_variant.variant.data["score_data"]["score"], # type: ignore - specifiedBy=publication_identifiers_to_method( - mapped_variant.variant.score_set.publication_identifier_associations - ), - sourceDataSet=score_set_to_data_set(mapped_variant.variant.score_set), + description=f"Variant effect study result for {context.variant.urn}.", + # post_mapped is guaranteed non-null by variant_annotation_context (it returns None otherwise). + focusVariant=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] + functionalImpactScore=variant_score(context.variant), + specifiedBy=publication_identifiers_to_method(context.variant.score_set.publication_identifier_associations), + sourceDataSet=score_set_to_data_set(context.variant.score_set), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), - mavedb_creator_contribution(mapped_variant.variant, mapped_variant.variant.score_set.created_by), - mavedb_modifier_contribution(mapped_variant.variant, mapped_variant.variant.score_set.modified_by), + mavedb_vrs_contribution(context), + mavedb_creator_contribution(context.variant, context.variant.score_set.created_by), + mavedb_modifier_contribution(context.variant, context.variant.score_set.modified_by), ], - reportedIn=filter(None, [variant_as_iri(mapped_variant.variant), mapped_variant_as_iri(mapped_variant)]), + reportedIn=filter(None, [variant_as_iri(context.variant), measured_allele_as_iri(context.measured_allele)]), ) diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py deleted file mode 100644 index 23ae95b66..000000000 --- a/src/mavedb/lib/annotation/util.py +++ /dev/null @@ -1,571 +0,0 @@ -from typing import Iterable, Literal, Optional - -from ga4gh.core.models import Extension -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided -from ga4gh.vrs.models import ( - Allele, - CisPhasedBlock, - Expression, - LengthExpression, - LiteralSequenceExpression, - MolecularVariation, - ReferenceLengthExpression, - SequenceLocation, - SequenceReference, -) - -from mavedb.lib.annotation.classification import ( - ExperimentalVariantFunctionalImpactClassification, - functional_classification_of_variant, - pathogenicity_classification_of_variant, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata -from mavedb.lib.permissions.principal import Principal -from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_calibration import ScoreCalibration -from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification - -CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" -"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" - - -def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: - """ - Converts a dictionary containing allelic mapping results into an Allele object. - - This function handles the possibility of an extra nesting level in early VRS 1.3 objects, - where Allele objects are contained within a `variation` property. If the `variation` key - is not present, the function assumes the dictionary itself represents the variation. - - Args: - allelic_mapping_results (dict): A dictionary containing allelic mapping results. - It may include a `variation` key or directly represent the variation. - - Returns: - Allele: An Allele object constructed from the provided mapping results. - - Raises: - KeyError: If required keys are missing from the input dictionary. - """ - - # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects - # are contained in a `variation` property. Although it's unlikely variants of this form - # will ever be exported in this format, we handle the possibility. - try: - variation = allelic_mapping_results["variation"] - except KeyError: - variation = allelic_mapping_results - - state_dict = variation["state"] - state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression - if state_dict.get("type") == "ReferenceLengthExpression": - state = ReferenceLengthExpression(**state_dict) - elif state_dict.get("type") == "LengthExpression": - state = LengthExpression(**state_dict) - elif state_dict.get("type") == "LiteralSequenceExpression": - state = LiteralSequenceExpression(**state_dict) - else: - raise ValueError( - f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " - "Update allele_from_mapped_variant_dictionary_result to handle this type." - ) - - # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on - # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid, so using model_validate() - # directly is not possible against stored mapping results. - return Allele( - id=variation.get("id"), - state=state, - digest=variation.get("digest"), - location=SequenceLocation( - start=variation.get("location", {}).get("start"), - end=variation.get("location", {}).get("end"), - digest=variation.get("location", {}).get("digest"), - id=variation.get("location", {}).get("id"), - sequenceReference=SequenceReference( - name=variation.get("location", {}).get("sequenceReference", {}).get("name"), - refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), - ), - ), - extensions=[ - Extension( - id=extension.get("id"), - name=extension["name"], - description=extension.get("description"), - value=extension.get("value"), - ) - for extension in variation.get("extensions", []) - ], - expressions=[ - Expression( - id=expression.get("id"), - syntax=expression["syntax"], - syntax_version=expression.get("syntax_version"), - value=expression["value"], - ) - for expression in variation.get("expressions", []) - ], - ) - - -def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: - """ - Extracts a VRS (Variation Representation Specification) object from a mapped variant. - - This function processes a dictionary of mapping results and returns a VRS object, - which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object - returned depends on the "type" field in the input dictionary. - - Args: - mapping_results (dict): A dictionary containing the mapping results of a variant. - It must include a "type" key indicating the type of VRS object - ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). - If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also - include a "members" key containing a list of member alleles. - - Returns: - MolecularVariation: A VRS object representing the mapped variant. This will be - either a `CisPhasedBlock` containing its member variants or an `Allele` - derived from the mapping results. - - Raises: - KeyError: If required keys are missing from the `mapping_results` dictionary. - """ - if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": - return MolecularVariation( - # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) - CisPhasedBlock( - id=None, - members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], - ) - ) - - return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) - - -def variation_from_mapped_variant(mapped_variant: MappedVariant) -> MolecularVariation: - """ - Converts mapping results from a mapped variant into a MolecularVariation object. - - This function takes a `MappedVariant` object and extracts its post-mapped - variant to generate a corresponding `MolecularVariation` object. If the - `post_mapped` attribute of the `MappedVariant` is `None`, an exception is raised. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing - the variant data. - - Returns: - MolecularVariation: The molecular variation derived from the post-mapped - variant. - - Raises: - MappingDataDoesntExistException: If the `post_mapped` attribute of the - `mapped_variant` is `None`, indicating that the post-mapped variant - data is unavailable. - """ - if mapped_variant.post_mapped is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have a post mapped variant." - " Unable to extract variation data." - ) - - return vrs_object_from_mapped_variant(mapped_variant.post_mapped) - - -def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> bool: - """ - Check if a mapped variant meets the basic requirements for annotation. - - This function validates that a mapped variant has the necessary data - to proceed with annotation by checking for a valid score value. - - Args: - mapped_variant (MappedVariant): The mapped variant to check for - annotation eligibility. - - Returns: - bool: True if the variant can be annotated (has score ranges and - a non-None score), False otherwise. - """ - # This property is guaranteed to exist for all variants. - if mapped_variant.variant.data["score_data"]["score"] is None: # type: ignore - return False - - return True - - -def score_calibration_may_be_used_for_annotation( - score_calibration: ScoreCalibration, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a score calibration may be used for annotation based on its properties. - - This function evaluates whether a given score calibration is suitable for use in - annotation by checking its research use only status and the presence of required - classifications based on the annotation type. - - Args: - score_calibration (ScoreCalibration): The score calibration to evaluate. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation - being considered, which determines the required classifications for validity. - allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations - marked as research use only for annotation. Defaults to False. - - Returns: - bool: True if the score calibration can be used for annotation, False otherwise. - """ - if score_calibration.research_use_only and not allow_research_use_only_calibrations: - return False - - if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: - return False - - if annotation_type == "pathogenicity" and all( - fr.acmg_classification is None for fr in score_calibration.functional_classifications - ): - return False - - return True - - -def calibrations_available_for_annotation( - mapped_variant: MappedVariant, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, - principal: Optional[Principal] = None, -) -> list[ScoreCalibration]: - """ - Select the calibrations on a mapped variant's score set that may build the requested annotation. - - Two independent questions decide this, and they are asked by different collaborators. - - Eligibility: Does this calibration carry the classifications the annotation type needs, and is - its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. - - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule - is stricter than its score set's. - - Args: - mapped_variant (MappedVariant): The mapped variant whose score set's calibrations are considered. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller, so a function that omits it gets public calibrations only. - - Returns: - list[ScoreCalibration]: The eligible, visible calibrations, in score set order. - """ - viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) - - return [ - score_calibration - for score_calibration in viewer.visible(mapped_variant.variant.score_set.score_calibrations) - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type, - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ) - ] - - -def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: - """ - Describe the principal an annotation was built for, given the calibrations behind it. - - VA-Spec statements carry no stable identifier, so two callers can receive materially different - statements from the same URL. Naming the scope on the object itself is what keeps that honest: a - consumer holding a record can tell whether it is the one anyone would get, or one widened by the - requester's own access. - - Args: - calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. - - Returns: - Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing - calibration is private and ``public`` otherwise. - """ - if any(calibration.private for calibration in calibrations): - return Extension( - name=CALIBRATION_SCOPE_EXTENSION_NAME, - value="restricted", - description=( - "Built from at least one private score calibration, visible to the requesting viewer. " - "Another viewer requesting this variant may receive fewer evidence lines, or none." - ), - ) - - return Extension( - name=CALIBRATION_SCOPE_EXTENSION_NAME, - value="public", - description=( - "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." - ), - ) - - -def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, - allow_research_use_only_calibrations=False, - principal: Optional[Principal] = None, -) -> bool: - """ - Determine if a mapped variant can be annotated for pathogenicity evidence. - - This function checks if a variant meets all the necessary conditions to receive - pathogenicity evidence annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for pathogenicity evidence annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object to evaluate - for pathogenicity evidence annotation eligibility. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller. Must match the principal the annotation itself will be built for, or this answers a - different question than the one the caller is about to act on. - - Returns: - bool: True if the variant can be annotated for pathogenicity evidence, - False otherwise. - - Notes: - The function performs two main validation checks: - 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. - - Both checks must pass for the variant to be considered eligible for - pathogenicity evidence annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - - return bool( - calibrations_available_for_annotation( - mapped_variant, - "pathogenicity", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - principal=principal, - ) - ) - - -def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, - allow_research_use_only_calibrations=False, - principal: Optional[Principal] = None, -) -> bool: - """ - Determine if a mapped variant can be annotated for functional statements. - - This function checks if a variant meets all the necessary conditions to receive - functional annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for functional annotation. - - Args: - mapped_variant (MappedVariant): The variant object to check for annotation - eligibility, containing mapping information and score data. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller. Must match the principal the annotation itself will be built for, or this answers a - different question than the one the caller is about to act on. - - Returns: - bool: True if the variant can be annotated for functional statements, - False otherwise. - - Notes: - The function performs two main checks: - 1. Validates base assumptions using _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for functional annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - - return bool( - calibrations_available_for_annotation( - mapped_variant, - "functional", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - principal=principal, - ) - ) - - -def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: - """ - Extract the sequence feature (e.g., gene or transcript) associated with a mapped variant. - - This function retrieves the sequence feature from the mapped variant's data, which is - necessary for generating annotations that reference specific genomic features. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant data. - - Returns: - SequenceFeature: Named tuple with: - - `identifier`: sequence feature identifier (gene/transcript ID or name) - - `system`: source/system URL for the identifier - - """ - target = target_for_variant(mapped_variant.variant) - if target is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable target gene." - " Unable to extract sequence feature for annotation." - ) - - # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. - # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or - # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. - if target.mapped_hgnc_name: - return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") - - post_mapped_ids = extract_ids_from_post_mapped_metadata( - target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore - ) - if post_mapped_ids: - post_mapped_id = post_mapped_ids[0] - if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): - return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") - elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): - return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") - - return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") - - if target.name: - return SequenceFeature(target.name, "https://www.mavedb.org/") - - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable sequence feature in its target gene data." - " Unable to extract sequence feature for annotation." - ) - - -def select_strongest_functional_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for functional classification. - - In case of ties or conflicting classifications, defaults to normal classification. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should be classified as INDETERMINATE but still receive annotations. - """ - if not calibrations: - return None, None - - # Collect all calibrations and their classifications - candidates: list[ - tuple[ - ScoreCalibration, - ScoreCalibrationFunctionalClassification, - ExperimentalVariantFunctionalImpactClassification, - ] - ] = [] - - for calibration in calibrations: - functional_range, classification = functional_classification_of_variant(mapped_variant, calibration) - if functional_range is not None: - candidates.append((calibration, functional_range, classification)) - - # If variant is not in any range, return first calibration with None to indicate INDETERMINATE - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Check if all classifications agree - classifications = [c[2] for c in candidates] - if all(cls == classifications[0] for cls in classifications): - # All agree, return the first one - return candidates[0][0], candidates[0][1] - - # Conflict exists: default to normal classification - normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] - if normal_candidates: - return normal_candidates[0][0], normal_candidates[0][1] - - # If no normal classification, return the first candidate - return candidates[0][0], candidates[0][1] - - -def select_strongest_pathogenicity_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for pathogenicity classification. - - Uses ACMG evidence strength to determine the strongest evidence. - In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain - significance by returning None for the functional range. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should receive annotations even though it's not classified in any range. - """ - if not calibrations: - return None, None - - # Define evidence strength ordering (higher index = stronger evidence) - # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. - # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. - strength_order = { - None: 0, - VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, - VaSpecStrengthOfEvidenceProvided.MODERATE: 2, - VaSpecStrengthOfEvidenceProvided.STRONG: 3, - VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, - } - - # Collect all calibrations with their evidence strength and criterion - candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] - - for calibration in calibrations: - functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, calibration - ) - if functional_range is not None: - strength_value = strength_order.get(evidence_strength, 0) - is_benign = criterion.name.startswith("B") if criterion else False - candidates.append((calibration, functional_range, strength_value, is_benign)) - - # If variant is not in any range, return first calibration with None - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Find the maximum strength - max_strength = max(c[2] for c in candidates) - strongest_candidates = [c for c in candidates if c[2] == max_strength] - - # If only one with max strength, return it - if len(strongest_candidates) == 1: - return strongest_candidates[0][0], strongest_candidates[0][1] - - # Tie: check for conflicting evidence (both benign and pathogenic) - has_benign = any(c[3] for c in strongest_candidates) - has_pathogenic = any(not c[3] for c in strongest_candidates) - - # If there's a conflict between benign and pathogenic evidence of equal strength, - # return None for the functional range to indicate uncertain significance - if has_benign and has_pathogenic: - return strongest_candidates[0][0], None - - # Otherwise return the first of the strongest - return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation_status_manager.py b/src/mavedb/lib/annotation_status_manager.py index 6598bfabd..4022cb4d2 100644 --- a/src/mavedb/lib/annotation_status_manager.py +++ b/src/mavedb/lib/annotation_status_manager.py @@ -1,207 +1,170 @@ -"""Manage annotation statuses for variants. - -This module provides functionality to insert and retrieve annotation statuses -for genetic variants, ensuring that only one current status exists per -(variant, annotation type, version) combination. +"""Append-only writer for the annotation event log. + +Buffers :class:`AnnotationEvent` rows and flushes them in batches. This is an +**append-only** log, not a state table: there is no ``current`` flag to +maintain and no retire-on-write step. "Current" is derived at read time +(``DISTINCT ON (subject, annotation_type) … ORDER BY id DESC``) — exposed as +the ``v_current_annotation_events`` view (``mavedb.models.annotation_event_view``). + +Each event's *subject* is either a variant or an allele, chosen by +``annotation_type``. The writer validates the subject/type pairing up front so +a mis-subjected event fails with a clear ``ValueError`` rather than a deferred +DB ``CHECK`` violation at flush. """ import logging from typing import Optional -from sqlalchemy import select, update +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.sql import desc from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.annotation_event import ALLELE_SUBJECT_TYPES, VARIANT_SUBJECT_TYPES, AnnotationEvent logger = logging.getLogger(__name__) -# Default number of pending annotations to accumulate before auto-flushing. +# Default number of pending events to accumulate before auto-flushing. DEFAULT_BATCH_SIZE = 500 class AnnotationStatusManager: - """ - Manager for handling variant annotation statuses with batched writes. + """Buffered, append-only writer for :class:`AnnotationEvent` rows. - Annotations are accumulated in memory and flushed to the database in - batches (default 500) to reduce round-trips. Callers **must** call - :meth:`flush` after the last ``add_annotation`` to persist any remainder. + Events are accumulated in memory and flushed in batches (default 500) to + reduce round-trips. Callers **must** call :meth:`flush` after the last + :meth:`record_event` to persist any remainder. """ - def __init__(self, session: Session, job_run_id: Optional[int] = None, *, batch_size: int = DEFAULT_BATCH_SIZE): + def __init__( + self, + session: Session, + job_run_id: Optional[int] = None, + *, + score_set_id: Optional[int] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + ): self.session = session self.job_run_id = job_run_id + self.score_set_id = score_set_id self.batch_size = batch_size - self._pending: list[VariantAnnotationStatus] = [] - self._retirement_filters: list[dict] = [] + self._pending: list[AnnotationEvent] = [] - def add_annotation( + def record_event( self, - variant_id: int, annotation_type: AnnotationType, - status: AnnotationStatus, - version: Optional[str] = None, - failure_category: Optional[AnnotationFailureCategory] = None, - annotation_data: dict = {}, - current: bool = True, - replace_all_versions: bool = True, + *, + disposition: Disposition, + reason: str, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + metadata: Optional[dict] = None, + score_set_id: Optional[int] = None, ) -> None: - """ - Stage a new annotation and schedule retirement of previous current rows. - - By default (``replace_all_versions=True``), all existing current annotations for - (variant, type) are retired regardless of version. + """Buffer one terminal observation about a variant or an allele. - When ``replace_all_versions=False``, only existing current annotations matching - (variant, type, version) are retired. + Exactly one of ``variant_id`` / ``allele_id`` must be set, and it must + match the subject the ``annotation_type`` keys on. ``reason`` is the + single "what happened" axis across all dispositions (see EventReason). - Writes are accumulated in memory and flushed to the database when - ``batch_size`` is reached. Call :meth:`flush` after the last add to - persist any remaining annotations. - - NOTE: - This method does not commit the session. The caller is responsible - for persisting changes (e.g., via ``session.commit()``). + Writes are accumulated in memory and flushed when ``batch_size`` is + reached. Call :meth:`flush` after the last call to persist the + remainder. Does not commit — the caller owns the transaction. """ - self._retirement_filters.append( - { - "variant_id": variant_id, - "annotation_type": annotation_type, - "replace_all_versions": replace_all_versions, - "version": version, - } - ) + self._validate_subject(annotation_type, variant_id, allele_id) self._pending.append( - VariantAnnotationStatus( - variant_id=variant_id, + AnnotationEvent( annotation_type=annotation_type, - status=status, - version=version, - failure_category=failure_category, - current=current, + variant_id=variant_id, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + event_metadata=metadata, job_run_id=self.job_run_id, - **annotation_data, + score_set_id=score_set_id if score_set_id is not None else self.score_set_id, ) # type: ignore[call-arg] ) if len(self._pending) >= self.batch_size: self.flush() - def flush(self) -> None: - """Flush all pending annotations to the database. + @staticmethod + def _validate_subject(annotation_type: AnnotationType, variant_id: Optional[int], allele_id: Optional[int]) -> None: + if (variant_id is None) == (allele_id is None): + raise ValueError("Exactly one of variant_id or allele_id must be set") - Retires old ``current=True`` rows in bulk, then inserts all pending - new rows in a single ``add_all`` + ``flush``. This replaces the - previous pattern of 2 flushes per ``add_annotation`` call. - """ + type_value = annotation_type.value if isinstance(annotation_type, AnnotationType) else annotation_type + if variant_id is not None and type_value not in VARIANT_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is allele-subject; pass allele_id, not variant_id") + if allele_id is not None and type_value not in ALLELE_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is variant-subject; pass variant_id, not allele_id") + + def flush(self) -> None: + """Insert all pending events in a single ``add_all`` + ``flush``.""" if not self._pending: return - self._retire_existing() self.session.add_all(self._pending) self.session.flush() - logger.debug(f"Flushed {len(self._pending)} annotation statuses") + logger.debug(f"Flushed {len(self._pending)} variant events") self._pending.clear() - self._retirement_filters.clear() - - def _retire_existing(self) -> None: - """Bulk-retire existing current annotations for all pending writes. - - Groups retirement filters by (annotation_type, replace_all_versions, version) - and issues one UPDATE per group, minimizing round-trips. - """ - # Group filters to minimize UPDATE statements. - # Key: (annotation_type, replace_all_versions, version) -> list of variant_ids - groups: dict[tuple, list[int]] = {} - for f in self._retirement_filters: - key = (f["annotation_type"], f["replace_all_versions"], f["version"]) - groups.setdefault(key, []).append(f["variant_id"]) - - for (annotation_type, replace_all_versions, version), variant_ids in groups.items(): - conditions = [ - VariantAnnotationStatus.variant_id.in_(variant_ids), - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ] - if not replace_all_versions: - conditions.append(VariantAnnotationStatus.version == version) - - stmt = update(VariantAnnotationStatus).where(*conditions).values(current=False) - self.session.execute(stmt) def get_current_annotation( - self, variant_id: int, annotation_type: AnnotationType, version: Optional[str] = None - ) -> Optional[VariantAnnotationStatus]: - """ - Retrieve the current annotation for a given variant/type/version. - - Flushes pending annotations first to ensure the result is up to date. - """ - self.flush() - - stmt = select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) - - result = self.session.execute(stmt) - return result.scalar_one_or_none() - - def get_annotation_history( self, - variant_id: int, annotation_type: AnnotationType, - version: Optional[str] = None, - ) -> list[VariantAnnotationStatus]: - """ - Return the full annotation timeline for a variant/type, newest first. - - Includes both current and retired rows — useful for debugging and - support investigations. + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> Optional[AnnotationEvent]: + """Latest event for a single ``(subject, annotation_type)`` key. + + Current status is the newest event by ``id`` — there is no ``current`` + flag to filter on. Flushes pending events first so the result reflects + buffered writes. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - ) - .order_by(desc(VariantAnnotationStatus.id)) - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) - return list(self.session.scalars(stmt).all()) + stmt = stmt.order_by(desc(AnnotationEvent.id)).limit(1) + return self.session.scalars(stmt).first() - def get_all_current_annotations( + def get_event_history( self, - variant_id: int, - ) -> list[VariantAnnotationStatus]: - """ - Return all current annotations for a variant, across all types and versions. - - Useful for a quick overview of what annotations are active for a given variant. + annotation_type: AnnotationType, + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> list[AnnotationEvent]: + """Full event timeline for a ``(subject, annotation_type)`` key, newest first. + + The append-only log retains every observation — skips, reconfirms, + no-ops — so this is the complete audit trail, not just the current row. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.current.is_(True), - ) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.version) - ) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) + stmt = stmt.order_by(desc(AnnotationEvent.id)) return list(self.session.scalars(stmt).all()) diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py new file mode 100644 index 000000000..b551472a2 --- /dev/null +++ b/src/mavedb/lib/cat_vrs.py @@ -0,0 +1,323 @@ +"""Cat-VRS transit, built on the fly from a variant's live allele links. + +The Categorical Variant served on ``/variants/{urn}`` is assembled per request from the variant's +live ``MappingRecordAllele`` links (see ``lib/alleles.py::get_live_record_allele_links``). +""" + +import logging +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept, Relation +from ga4gh.core.models import ConceptMapping, iriReference +from ga4gh.core.models import Relation as MappingRelation +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.logging.context import logging_context +from mavedb.lib.term_systems import GKS_ALLELE_RELATION, MAVEDB_CAT_VRS_RELATION, SEQUENCE_ONTOLOGY, TermSystem +from mavedb.lib.term_systems import coding as term_coding +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + +logger = logging.getLogger(__name__) + +# Cat-VRS does not attribute its own Relation values to one uniform system: translation_of and +# transcribed_to are Sequence Ontology terms, while liftover_to has no ontology equivalent and is +# carried on Cat-VRS's own internal vocabulary instead. Per-term, not a single constant, so that +# extending `_SPEC_EQUIVALENT` can't silently mislabel a future mapping with the wrong system. +# +# Sourced from the spec's published examples/schema (ga4gh/cat-vrs recipes-source.yaml and +# examples/json/proteinSequenceConsequence-ex2.json). +_SPEC_TERM_SYSTEM: dict[Relation, TermSystem] = { + Relation.TRANSLATION_OF: SEQUENCE_ONTOLOGY, + Relation.TRANSCRIBED_TO: SEQUENCE_ONTOLOGY, + Relation.LIFTOVER_TO: GKS_ALLELE_RELATION, +} + + +class CatVrsRelation(str, Enum): + """Relation of a member allele to the single defining (measured) allele. + + Deliberately **not** a subclass of Cat-VRS's ``Relation``: Python forbids extending an enum that + already has members, and inheriting would buy nothing anyway. Interop here is carried by + ``Coding.system`` on the emitted concept, not by Python enum identity — see + :func:`_relation_concept`, which maps the codes that have a spec equivalent onto it. Keeping this a + plain enum keeps the members statically visible to mypy and greppable. + """ + + # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. + ENCODES = "encodes" + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" + # defining is nt; member is the protein consequence. Consequence, no independent score. + TRANSLATION_OF = "translation_of" + # defining is nt; member is a *convergent encoding* — an nt allele in a different projection group that + # encodes the same protein consequence as the measured change. + CO_ENCODES = "co_encodes" + + +# MaveDB code -> the Cat-VRS term it is an exact match for. A code with no entry has no spec equivalent. +_SPEC_EQUIVALENT: dict[CatVrsRelation, Relation] = { + CatVrsRelation.TRANSLATION_OF: Relation.TRANSLATION_OF, +} + + +class CatVrsMode(str, Enum): + """The score-collapse semantics of the categorical variant.""" + + PROJECTION = "projection" # Mode 1 — nt measured; score rides faithfully. + REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. + + +@dataclass +class CategoricalVariantTransit: + """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" + + categorical_variant: CategoricalVariant + mode: CatVrsMode + member_relations: dict[str, CatVrsRelation] + """vrs_digest keyed dict encoding digest: relation for the member -> defining relationship. + Excludes the defining allele itself. + """ + + +def is_convergent_encoding( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """Whether a projection-mode member is a *convergent encoding* of the measured change. + + Concretely: + - The defining (measured) allele is nt (genomic or cdna). + - The member is nt (genomic or cdna). + - The member is in a different projection group than the defining (measured) allele. + """ + return ( + member_level in NUCLEOTIDE_LEVELS + and member_group is not None + and defining_group is not None + and member_group != defining_group + ) + + +def _relation_for( + defining_level: Optional[str], + member_level: Optional[str], + *, + defining_group: Optional[int] = None, + member_group: Optional[int] = None, +) -> Optional[CatVrsRelation]: + """Derive the member -> defining relation from the two levels, or None for the defining itself. + + Star model: every member relates to the single defining allele, so the relation depends only on the + (defining, member) level pair. Note that in projection mode relation also depends on whether the nt + member shares the measured change's ``projection_group`` (its projection) or lives in another group + (a convergent encoding). + """ + # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. + if defining_level == SequenceLevel.protein.value: + return CatVrsRelation.ENCODES if member_level in NUCLEOTIDE_LEVELS else None + + # Nt measured (genomic or cdna). + if member_level == SequenceLevel.protein.value: + return CatVrsRelation.TRANSLATION_OF + if member_level in NUCLEOTIDE_LEVELS: + if is_convergent_encoding(member_level, member_group, defining_group=defining_group): + return CatVrsRelation.CO_ENCODES + return CatVrsRelation.COORDINATE_REPRESENTATION_OF + + # Unreachable in practice. Levels are constrained by the DB and the mapping job. + return None # pragma: no cover + + +def _is_precise_projection_member( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """In projection mode, whether a member is a *precise* representation of the measured change. + + True for the protein consequence (the apex, shared by the whole equivalence class) and the measured + change's own projection. False for convergent encodings in other projection groups. + """ + if member_level == SequenceLevel.protein.value: + return True + + return not is_convergent_encoding(member_level, member_group, defining_group=defining_group) + + +def _relation_concept(relation: CatVrsRelation) -> MappableConcept: + """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept``. + + The MaveDB code is always the ``primaryCoding``. Codes are read *member -> defining*, which is + MaveDB's own framing, so claiming the spec's system for them outright would misrepresent their + provenance. Where a code *is* the same concept as a published Cat-VRS relation, an ``exactMatch`` + :class:`ConceptMapping` to that term is attached via ``MappableConcept.mappings``. + + A code with no spec equivalent carries no mapping. See :data:`_SPEC_EQUIVALENT` for which spec + gaps exist and why. + """ + spec_term = _SPEC_EQUIVALENT.get(relation) + return MappableConcept( + name=relation.value, + primaryCoding=term_coding(MAVEDB_CAT_VRS_RELATION, relation.value), + mappings=( + [ + ConceptMapping( + coding=term_coding(_SPEC_TERM_SYSTEM[spec_term], spec_term.value), + relation=MappingRelation.EXACT_MATCH, + ) + ] + if spec_term is not None + else None + ), + ) + + +def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: + """Bare VRS variation (Allele or CisPhasedBlock) from the stored post_mapped JSONB. + + ``None`` when the allele has no post_mapped representation — it cannot be a Cat-VRS member. + """ + if allele.post_mapped is None: + return None + + variation = vrs_object_from_mapped_variant(allele.post_mapped).root + # post_mapped only ever holds an Allele or CisPhasedBlock (the two shapes + # vrs_object_from_mapped_variant produces); MolecularVariation.root is a wider union, so narrow. + assert isinstance(variation, (VrsAllele, CisPhasedBlock)) + return variation + + +def build_categorical_variant( + links: list[MappingRecordAllele], *, name: str, include_convergent: bool = True +) -> Optional[CategoricalVariantTransit]: + """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. + + ``links`` is the record-scoped live link set from ``get_live_record_allele_links``. Exactly one + is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` + when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + + **The categorical variant always anchors on the measured allele**. Member selection therefore differs by mode: + + - **Reverse translation** (protein measured): the defining allele is the protein change and every + nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled. That class + is exactly what the protein measurement's claim ranges over. ``include_convergent`` is inert here. + - **Projection** (nt measured): the record also carries the reverse-translation fan out. This includes + all synonymous *convergent encodings* (other nt alleles encoding the same protein consequence, in + different projection groups), which are distinct, unmeasured variants rather than representations + of the measured change. ``include_convergent`` selects how they are handled: + + - ``True`` (default; the detail envelope): the encodings are kept as members wearing the + :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's + projection and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. + - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`_is_precise_projection_member`) + and only the measured change's precise projection and protein consequence remain. + """ + defining_link = next((link for link in links if link.is_authoritative), None) + if defining_link is None: + return None + + defining_allele = defining_link.allele + defining_vrs = _hydrate_vrs(defining_allele) + # The authoritative (measured) allele having no post_mapped breaks an invariant the mapping + # job upholds. Returning None here is analogous to "unmapped". + if defining_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: authoritative allele {defining_allele.vrs_digest!r} has no " + "post_mapped representation; treating the variant as unmapped." + ), + extra=logging_context(), + ) + return None + + defining_level = defining_allele.level + mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == SequenceLevel.protein.value else CatVrsMode.PROJECTION + + members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] + member_relations: dict[str, CatVrsRelation] = {} + relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds + + for link in links: + if link is defining_link: + continue + + allele = link.allele + + # Projection mode, narrow object (include_convergent=False): drop any convergent encodings the + # reverse-translation fan left on the record, keeping only the measured change's precise coordinate + # projection and its protein consequence. + if ( + mode is CatVrsMode.PROJECTION + and not include_convergent + and not _is_precise_projection_member( + allele.level, link.projection_group, defining_group=defining_link.projection_group + ) + ): + continue + + member_vrs = _hydrate_vrs(allele) + # A live link to an allele with no post_mapped is an unexpected data state. + if member_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: skipping member allele {allele.vrs_digest!r} with no " + "post_mapped representation." + ), + extra=logging_context(), + ) + continue + + members.append(member_vrs) + relation = _relation_for( + defining_level, + allele.level, + defining_group=defining_link.projection_group, + member_group=link.projection_group, + ) + if relation is not None and allele.vrs_digest is not None: + member_relations[allele.vrs_digest] = relation + relations_present[relation] = None + + # The defining allele anchors the DefiningAlleleConstraint. Cat-VRS 1.0.0 requires a bare + # vrs:Allele (or an iri ref) there, so a multi-variant defining (CisPhasedBlock) is referenced by + # its digest instead. + defining_ref: VrsAllele | iriReference = ( + defining_vrs if isinstance(defining_vrs, VrsAllele) else iriReference(root=defining_allele.vrs_digest or "") + ) + + categorical_variant = CategoricalVariant( + name=name, + members=members, + constraints=[ + DefiningAlleleConstraint( + allele=defining_ref, + relations=[_relation_concept(relation) for relation in relations_present] or None, + ) + ], + ) + + return CategoricalVariantTransit( + categorical_variant=categorical_variant, + mode=mode, + member_relations=member_relations, + ) + + +def categorical_variant_for_variant( + db: Session, variant_id: int, *, name: str, as_of: Optional[datetime] = None +) -> Optional[CategoricalVariantTransit]: + """Fetch a variant's live allele links and assemble its Cat-VRS transit object. + + The DB-backed entry point routers use. ``as_of`` is a fetch concern, threaded into the link query + only: membership is what varies over time, while each allele's VRS is immutable and content-addressed, + so :func:`build_categorical_variant` stays pure and time-agnostic over the links it is handed. Returns + ``None`` for an unmapped variant (no authoritative link at ``as_of``). + """ + links = get_live_record_allele_links(db, variant_id, as_of=as_of) + return build_categorical_variant(links, name=name) diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index b773e6891..b28a184d1 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -136,80 +136,6 @@ async def get_associated_clinvar_allele_id(clingen_allele_id: str) -> str: return "" -def extract_hgvs_from_ca_allele_data( - data: dict, - target_is_coding: bool, - transcript_accession: Optional[str], -) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a CA (canonical allele) ID. - - Parses the ClinGen API response to find GRCh38 genomic HGVS, coding HGVS - matching the target transcript (or MANE fallback), and protein HGVS. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - target_is_coding: Whether the score set target is protein-coding. - transcript_accession: Specific transcript accession to match, or None to use MANE. - - Returns: - Tuple of (hgvs_g, hgvs_c, hgvs_p), any of which may be None. - """ - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - if data.get("genomicAlleles"): - for allele in data["genomicAlleles"]: - if allele.get("referenceGenome") == "GRCh38" and allele.get("hgvs"): - hgvs_g = allele["hgvs"][0] - break - - if target_is_coding and data.get("transcriptAlleles"): - if transcript_accession: - for allele in data["transcriptAlleles"]: - if allele.get("hgvs"): - for hgvs_string in allele["hgvs"]: - hgvs_reference_sequence = hgvs_string.split(":")[0] - if transcript_accession == hgvs_reference_sequence: - hgvs_c = hgvs_string - break - if hgvs_c: - if allele.get("proteinEffect"): - hgvs_p = allele["proteinEffect"].get("hgvs") - break - else: - # No transcript specified; use MANE if available - for allele in data["transcriptAlleles"]: - if allele.get("MANE"): - hgvs_c = allele["MANE"].get("nucleotide", {}).get("RefSeq", {}).get("hgvs") - hgvs_p = allele["MANE"].get("protein", {}).get("RefSeq", {}).get("hgvs") - break - - return hgvs_g, hgvs_c, hgvs_p - - -def extract_hgvs_from_pa_allele_data(data: dict) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a PA (protein allele) ID. - - For PA alleles, only hgvs_p is extracted from aminoAcidAlleles. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - - Returns: - Tuple of (None, None, hgvs_p), where hgvs_p may be None. - """ - hgvs_p: Optional[str] = None - - if data.get("aminoAcidAlleles"): - for allele in data["aminoAcidAlleles"]: - if allele.get("hgvs"): - hgvs_p = allele["hgvs"][0] - break - - return None, None, hgvs_p - - def expand_allele_ids(clingen_allele_ids: list[Optional[str]]) -> set[str]: """Expand comma-separated multi-variant ClinGen allele IDs into individual IDs. diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py new file mode 100644 index 000000000..562dab94e --- /dev/null +++ b/src/mavedb/lib/clingen/alleles.py @@ -0,0 +1,111 @@ +"""Query helpers for fetching score-set alleles for ClinGen registration. + +Both submit_score_set_mappings_to_car and warm_clingen_cache use the same allele +scope: all current MappingRecordAllele links (authoritative and RT-derived) for a +score set. A single definition here prevents the two jobs from drifting apart. +""" + +from typing import Callable, Iterable, NamedTuple, Optional, TypeVar + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +P = TypeVar("P") + + +class ScoreSetAlleleRow(NamedTuple): + """One (allele, variant) link for a score set. An allele shared by multiple variants appears once + per variant; :func:`group_alleles_for_annotation` collapses those duplicates into one work-unit + per allele. + + ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the + VEP job can build its HGVS payload without a second query. ``level`` is the allele's sequence + level (``protein``/``cdna``/``genomic``), carried so jobs annotating just nucleotide alleles can + skip protein alleles without a second query. All are optional with a ``None`` default so payloads keying + only on the CAID need not name them; real DB rows always populate them. + """ + + allele_id: int + post_mapped: dict | None + clingen_allele_id: str | None + variant_id: int + hgvs_g: str | None = None + hgvs_c: str | None = None + hgvs_p: str | None = None + level: str | None = None + + +def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: + """Return all current alleles for a score set with their linked variant IDs. + + Covers both authoritative mapper alleles and RT-derived equivalence alleles — + the full set that requires ClinGen registration before the annotation fan-out + can run. + + Only alleles with a non-null ``post_mapped`` are returned — variants that failed + or were benignly absent have no allele link and cannot receive a CAID. + """ + rows = db.execute( + select( + Allele.id, + Allele.post_mapped, + Allele.clingen_allele_id, + Variant.id.label("variant_id"), + Allele.hgvs_g, + Allele.hgvs_c, + Allele.hgvs_p, + Allele.level, + ) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(Allele.post_mapped.is_not(None)) + ).all() + + return [ + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p, r.level) + for r in rows + ] + + +def group_alleles_for_annotation( + rows: Iterable[ScoreSetAlleleRow], + payload: Callable[[ScoreSetAlleleRow], Optional[P]], +) -> dict[int, P]: + """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one + job-specific payload per allele, keyed by ``allele_id``. + + The same allele recurs once per variant that links it, so this dedups those rows down to one + entry per allele — the shape every allele-keyed annotation job wants now that annotation events + are allele-keyed (one event per allele, never fanned per-variant). + + ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for + gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it + carries no CAID), replacing each job's ad-hoc ``if row.x is None: continue``. ``payload`` must be + a pure function of allele-level fields so its result is stable across an allele's rows. + + Grouping on ``allele_id`` rather than ``vrs_digest`` is intentional: the two are 1:1 (content + addressing makes ``vrs_digest`` unique), so the groups are identical either way, and ``allele_id`` + is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry + it on the row and store against it — the grouping contract here does not change. + """ + groups: dict[int, P] = {} + for row in rows: + if row.allele_id in groups: + continue + + built = payload(row) + if built is None: + continue + + groups[row.allele_id] = built + + return groups diff --git a/src/mavedb/lib/clingen/constants.py b/src/mavedb/lib/clingen/constants.py index 2276ee187..60f3a26cb 100644 --- a/src/mavedb/lib/clingen/constants.py +++ b/src/mavedb/lib/clingen/constants.py @@ -14,8 +14,17 @@ LDH_ENTITY_ENDPOINT = "maveDb" # for some reason, not the same :/ DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100 +DEFAULT_CAR_SUBMISSION_BATCH_SIZE = int(os.getenv("CAR_SUBMISSION_BATCH_SIZE", "10000")) +"""Number of HGVS strings sent to the ClinGen Allele Registry per PUT. Reverse translation can +produce very large allele counts, so submissions are chunked rather than sent as a single payload.""" CLINGEN_CACHE_WARMING_CONCURRENCY = int(os.getenv("CLINGEN_CACHE_WARMING_CONCURRENCY", "5")) """Maximum number of concurrent requests to make to the ClinGen API when pre-warming the cache for mapped variants.""" + +CLINGEN_CONNECT_TIMEOUT = float(os.getenv("CLINGEN_CONNECT_TIMEOUT", "5")) +CLINGEN_READ_TIMEOUT = float(os.getenv("CLINGEN_READ_TIMEOUT", "300")) +CLINGEN_HTTP_TIMEOUT = (CLINGEN_CONNECT_TIMEOUT, CLINGEN_READ_TIMEOUT) +"""(connect, read) timeout for ClinGen HTTP calls. Fail fast on connect, with a generous read +budget because a large batch can take CAR a while to respond.""" LDH_SUBMISSION_ENDPOINT = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}" LDH_ACCESS_ENDPOINT = os.getenv("LDH_ACCESS_ENDPOINT", "https://genboree.org/ldh") LDH_MAVE_ACCESS_ENDPOINT = f"{LDH_ACCESS_ENDPOINT}/{LDH_ENTITY_NAME}/id" diff --git a/src/mavedb/lib/clingen/content_constructors.py b/src/mavedb/lib/clingen/content_constructors.py index 1f55437f4..ce62ebbbd 100644 --- a/src/mavedb/lib/clingen/content_constructors.py +++ b/src/mavedb/lib/clingen/content_constructors.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from uuid import uuid4 from urllib.parse import quote_plus @@ -7,7 +6,8 @@ from mavedb.constants import MAVEDB_BASE_GIT, MAVEDB_FRONTEND_URL from mavedb.lib.types.clingen import LdhContentLinkedData, LdhContentSubject, LdhEvent, LdhSubmission from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.variant import Variant @@ -32,8 +32,12 @@ def construct_ldh_submission_subject(hgvs: str) -> LdhContentSubject: return {"Variant": {"hgvs": hgvs}} -def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[MappedVariant]) -> LdhContentLinkedData: - entity: LdhContentLinkedData = { +def construct_ldh_submission_entity( + variant: Variant, mapping_record: MappingRecord, allele: Allele +) -> LdhContentLinkedData: + # Pre-mapped data and the mapping API version live on the per-variant MappingRecord; + # post-mapped data lives on the (cross-variant deduped) Allele. + return { # TODO#372: We try to make all possible fields that are non-nullable represented that way. "MaveDBMapping": [ { @@ -41,27 +45,25 @@ def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[M "mavedb_id": variant.urn, # type: ignore "score": variant.data["score_data"]["score"], # type: ignore "score_set_description": variant.score_set.short_description, # type: ignore + "pre_mapped": mapping_record.pre_mapped, + "post_mapped": allele.post_mapped, + "mapping_api_version": mapping_record.mapping_api_version, }, "entId": variant.urn, # type: ignore "entIri": f"{MAVEDB_FRONTEND_URL}/score-sets/{quote_plus(variant.score_set.urn)}?variant={quote_plus(variant.urn)}", # type: ignore } ] } - if mapped_variant is not None: - entity["MaveDBMapping"][0]["entContent"]["pre_mapped"] = mapped_variant.pre_mapped - entity["MaveDBMapping"][0]["entContent"]["post_mapped"] = mapped_variant.post_mapped - entity["MaveDBMapping"][0]["entContent"]["mapping_api_version"] = mapped_variant.mapping_api_version - return entity def construct_ldh_submission( - variant_content: list[tuple[str, Variant, Optional[MappedVariant]]], + variant_content: list[tuple[str, Variant, MappingRecord, Allele]], ) -> list[LdhSubmission]: content_submission: list[LdhSubmission] = [] - for hgvs, variant, mapped_variant in variant_content: + for hgvs, variant, mapping_record, allele in variant_content: subject = construct_ldh_submission_subject(hgvs) event = construct_ldh_submission_event(subject) - entity = construct_ldh_submission_entity(variant, mapped_variant) + entity = construct_ldh_submission_entity(variant, mapping_record, allele) content_submission.append( { diff --git a/src/mavedb/lib/clingen/services.py b/src/mavedb/lib/clingen/services.py index 085d9103f..c8e80259f 100644 --- a/src/mavedb/lib/clingen/services.py +++ b/src/mavedb/lib/clingen/services.py @@ -8,7 +8,7 @@ import requests from jose import jwt -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import CLINGEN_HTTP_TIMEOUT, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.types.clingen import ClinGenAllele, ClinGenSubmissionError, LdhSubmission from mavedb.lib.utils import batched @@ -80,6 +80,7 @@ def dispatch_submissions( response = requests.put( url=request_url, data="\n".join(content_submissions), + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() @@ -165,10 +166,10 @@ def authenticate(self) -> str: auth_url = f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth" auth_body = {"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD} - auth_response = requests.post(auth_url, json=auth_body) try: + auth_response = requests.post(auth_url, json=auth_body, timeout=CLINGEN_HTTP_TIMEOUT) auth_response.raise_for_status() - except requests.exceptions.HTTPError as exc: + except requests.exceptions.RequestException as exc: save_to_logging_context(format_raised_exception_info_as_dict(exc)) logger.error(msg="Failed to authenticate with Genboree services.", exc_info=exc, extra=logging_context()) raise exc @@ -221,16 +222,17 @@ def dispatch_submissions( logger.info(msg=f"Dispatching {len(submissions)} ldh submissions...", extra=logging_context()) for idx, content in enumerate(submissions): try: - logger.debug(msg=f"Dispatching submission {idx+1}.", extra=logging_context()) + logger.debug(msg=f"Dispatching submission {idx + 1}.", extra=logging_context()) response = requests.put( url=self.url, json=content, headers={"Authorization": f"Bearer {self.authenticate()}", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() submission_successes.append(response.json()) logger.info( - msg=f"Successfully dispatched ldh submission ({idx+1} / {len(submissions)}).", + msg=f"Successfully dispatched ldh submission ({idx + 1} / {len(submissions)}).", extra=logging_context(), ) diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py new file mode 100644 index 000000000..3f9fd41e0 --- /dev/null +++ b/src/mavedb/lib/clinical_controls.py @@ -0,0 +1,126 @@ +"""Live ClinVar-control lookups for a score set, over the allele-link substrate. + +A score set's clinical controls are the ClinVar assertions whose allele is live-linked to one of the +score set's variants, reached by walking ``ClinvarAlleleLink → Allele → MappingRecordAllele → +MappingRecord → Variant`` with every ``ValidTime`` hop constrained to the same instant (``as_of``, +defaulting to currently-live). Both serving endpoints — the controls list +(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) walk the chain +that is defined here. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional, TypeVar + +from sqlalchemy import Select, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +SelectT = TypeVar("SelectT", bound=Select[Any]) + + +@dataclass(frozen=True) +class ControlVariantLink: + """One (score-set variant, annotated allele) pair that a clinical control reaches. + + ``allele_digest`` is the VRS digest of the allele the control annotates. A client compares it to + the variant's authoritative ``assay_level_digest``: + - a match is a call *on the assayed level* (this call takes precedence) + - a mismatch is a call on an *encoding* level (this call is only used when the assayed level is + unannotated). + + Without the digest, the two are indistinguishable and precedence is unimplementable. + """ + + variant_urn: str + allele_digest: str + + +def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: + """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the + score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the + chain fans a control out across each variant it links to.""" + return ( + stmt.join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + + +def get_clinical_controls_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_name: Optional[str] = None, + db_version: Optional[str] = None, +) -> list[tuple[ClinvarControl, list[ControlVariantLink]]]: + """The live ClinVar controls for a score set, each paired with the score-set variants that link to + it. Every link carrying the digest of the allele the control annotates (see :class:`ControlVariantLink`). + Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). Controls come back + in first-seen order. Each control's links preserve query order.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl, Variant.urn.label("variant_urn"), Allele.vrs_digest.label("allele_digest")), + score_set_id, + as_of, + ) + if db_name is not None: + stmt = stmt.where(ClinvarControl.db_name == db_name) + if db_version is not None: + stmt = stmt.where(ClinvarControl.db_version == db_version) + + controls: dict[int, ClinvarControl] = {} + links_by_control: dict[int, list[ControlVariantLink]] = {} + + # A persisted variant reached through the join always carries a URN. Narrow the nullable column so the + # URN matches the view model's required `str`. TODO(#372) + for ctrl, variant_urn, allele_digest in db.execute(stmt).tuples(): + if variant_urn is None: + continue + + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + links_by_control[ctrl.id] = [] + links_by_control[ctrl.id].append(ControlVariantLink(variant_urn=variant_urn, allele_digest=allele_digest)) + + return [(ctrl, links_by_control[ctrl.id]) for ctrl in controls.values()] + + +def get_clinical_control_options( + db: Session, score_set_id: int, *, as_of: Optional[datetime] = None +) -> list[tuple[str, list[str]]]: + """The options facet: each control DB live-linked to the score set, paired with its available + versions newest-first.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl.db_name, ClinvarControl.db_version), score_set_id, as_of + ) + options: dict[str, list[str]] = {} + for db_name, db_version in db.execute(stmt): + options.setdefault(db_name, []).append(db_version) + + return [ + (db_name, sorted(versions, key=clinvar_version_sort_key, reverse=True)) for db_name, versions in options.items() + ] + + +def clinvar_version_sort_key(version: str) -> tuple[int, int]: + """Sort key for a ClinVar release string in ``MM_YYYY`` form, ordering by ``(year, month)``. + Unparseable versions sort to the bottom as ``(0, 0)`` rather than raising.""" + try: + month, year = version.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..fd80212be 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -26,6 +26,10 @@ backoff for throttling is unnecessary — a modest retry with short backoff suffices. """ -CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus") +CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus", "VariationID") """Only these fields are extracted from each ClinVar TSV row and cached. The full TSV has ~30 columns; trimming to only what we need shrinks the cached pickle from hundreds of MB to tens of MB and speeds up load times. + +VariationID is ClinVar's canonical public identifier (anchors the web UI / variation links); we keep it +alongside the AlleleID (the row key) so the link record can carry both. A row missing the column on an +older archival TSV degrades to None rather than failing the whole version's parse. """ diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..d72968da8 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -175,8 +175,11 @@ def _fetch_parse_and_cache( # as a list (which would be 1.5–2 GB for a modern TSV). with gzip.open(filename=buf, mode="rt") as f: reader = csv.DictReader(f, delimiter="\t") # type: ignore + # row.get (not row[field]) so a field absent from an older archival TSV schema yields + # None for that row rather than raising and discarding the whole version's parse. data: Dict[str, Dict[str, str]] = { - str(row["#AlleleID"]): {field: row[field] for field in CLINVAR_FIELDS_TO_KEEP} for row in reader + str(row["#AlleleID"]): {field: row.get(field) for field in CLINVAR_FIELDS_TO_KEEP} # type: ignore[misc] + for row in reader } finally: csv.field_size_limit(default_csv_field_size_limit) diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py index 3469d6d64..c037041b8 100644 --- a/src/mavedb/lib/csv/annotations.py +++ b/src/mavedb/lib/csv/annotations.py @@ -10,8 +10,8 @@ from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.csv.specs import CsvMappedRow from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_calibration_functional_classification_variant_association import ( @@ -74,7 +74,7 @@ def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> di def annotations_for_rows( db: Session, variants: Sequence[Variant], - mappings: Sequence[Optional[MappedVariant]], + mappings: Sequence[Optional[CsvMappedRow]], calibration_namespaces: dict[str, str], viewer: Optional[ScoreCalibrationViewer] = None, ) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: @@ -103,7 +103,7 @@ def annotations_for_rows( if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: annotations[namespace] = None else: - annotations[namespace] = flatten_annotation(mapping, calibration, contained) + annotations[namespace] = flatten_annotation(variant, calibration, contained) rows.append(annotations) return rows diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py index a086aea71..2634c14b3 100644 --- a/src/mavedb/lib/csv/columns.py +++ b/src/mavedb/lib/csv/columns.py @@ -16,12 +16,11 @@ parse_calibration_namespace, parse_clinvar_namespace, ) -from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.csv.specs import CORE_NAMESPACE, CsvMappedRow, RowSource, namespace_spec from mavedb.lib.mave.utils import NA_VALUE, NULL_VALUES from mavedb.lib.validation.constants.general import hgvs_columns -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant _OUTPUT_NULL_STRINGS = frozenset(value.lower() for value in NULL_VALUES if value) @@ -133,9 +132,9 @@ def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: b def variant_to_csv_row( variant: Variant, columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, + mapping: Optional[CsvMappedRow] = None, gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, match_type: Optional[str] = None, namespaced: bool = False, @@ -189,9 +188,9 @@ def variant_to_csv_row( def variants_to_csv_rows( variants: Sequence[Variant], columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + mappings: Optional[Sequence[Optional[CsvMappedRow]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, match_types: Optional[Sequence[Optional[str]]] = None, namespaced: bool = False, @@ -199,9 +198,9 @@ def variants_to_csv_rows( ) -> Iterable[dict[str, Any]]: """Format each variant into a dictionary row containing the keys specified in *columns*.""" n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _mappings: Sequence[Optional[CsvMappedRow]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n ) _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py index ba38b7f60..113750ff5 100644 --- a/src/mavedb/lib/csv/entries.py +++ b/src/mavedb/lib/csv/entries.py @@ -6,12 +6,13 @@ """ from dataclasses import dataclass +from datetime import datetime from typing import Iterable, Optional, Sequence from sqlalchemy import and_, select from sqlalchemy.orm import Session -from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.annotation.calibration import score_calibration_may_be_used_for_annotation from mavedb.lib.csv.namespaces import ( CLINVAR_DB_NAME, STATIC_CSV_NAMESPACE_LABELS, @@ -22,8 +23,10 @@ clinvar_namespace_sort_key, ) from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -142,8 +145,10 @@ def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> l return entries -def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: - """Whether any variant in these score sets has a current mapping. +def score_sets_have_current_mappings( + db: Session, score_set_ids: Sequence[int], *, as_of: Optional[datetime] = None +) -> bool: + """Whether any variant in these score sets has a live mapping record with an authoritative allele. Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV should offer the namespaces, even if the variant in question is unmapped. @@ -153,16 +158,25 @@ def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) return ( db.scalars( - select(MappedVariant.id) - .join(MappedVariant.variant) - .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + select(MappingRecordAllele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ) + ) .limit(1) ).first() is not None ) -def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: +def clinvar_release_namespaces( + db: Session, score_set_ids: Sequence[int], *, as_of: Optional[datetime] = None +) -> list[str]: """Every ClinVar release namespace these score sets have data for. Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — @@ -173,14 +187,18 @@ def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> lis return [] db_versions = db.scalars( - select(ClinicalControl.db_version) - .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) - .join(MappedVariant.variant) + select(ClinvarControl.db_version) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == ClinvarAlleleLink.allele_id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) .where( and_( Variant.score_set_id.in_(score_set_ids), - MappedVariant.current.is_(True), - ClinicalControl.db_name == CLINVAR_DB_NAME, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ClinvarAlleleLink.live_at(as_of), + ClinvarControl.db_name == CLINVAR_DB_NAME, ) ) .distinct() diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py index 9244db15c..d4145541f 100644 --- a/src/mavedb/lib/csv/fetch.py +++ b/src/mavedb/lib/csv/fetch.py @@ -5,28 +5,195 @@ """ from dataclasses import dataclass +from datetime import datetime from typing import Any, Optional, Sequence -from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy import Integer, Select, and_, cast, func, select from sqlalchemy.orm import Session, aliased, selectinload from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME -from mavedb.lib.csv.specs import namespace_spec -from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.lib.csv.specs import CsvMappedRow, namespace_spec +from mavedb.lib.gnomad import GNOMAD_DB_NAME +from mavedb.lib.score_set_variants import get_protein_hgvs_by_record, mapped_hgvs_by_level +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +# CSV rows come out in variant-number order (the integer after '#'); id breaks ties stably. +_VARIANT_NUMBER_ORDER = (cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) @dataclass class CsvFetchResult: variants: list[Variant] - mappings: Optional[list[Optional[MappedVariant]]] + mappings: Optional[list[Optional[CsvMappedRow]]] gnomad_data: Optional[list[Optional[GnomADVariant]]] - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] + + +def _scope_query( + query: Select[Any], *, score_set_id: Optional[int], variant_ids: Optional[Sequence[int]] +) -> Select[Any]: + """Narrow *query* to one score set (ordered by variant number) or an explicit variant set.""" + if score_set_id is not None: + return query.where(Variant.score_set_id == score_set_id).order_by(*_VARIANT_NUMBER_ORDER) + return query.where(Variant.id.in_(variant_ids or [])) + + +def _restore_variant_id_order(rows: list[Any], variant_ids: Sequence[int], key) -> list[Any]: + """Postgres does not preserve IN-list order, so restore the caller's ordering by variant id.""" + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + return sorted(rows, key=lambda row: position.get(key(row), len(position))) + + +def _apply_pagination(query: Select[Any], start: Optional[int], limit: Optional[int]) -> Select[Any]: + """Apply the shared ``start``/``limit`` window (offset/limit) to a variant query. A falsy ``start`` or + ``limit`` leaves that bound off.""" + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + return query + + +def _substrate_query( + *, score_set_id: Optional[int], variant_ids: Optional[Sequence[int]], as_of: Optional[datetime] +) -> Select[Any]: + """The base per-variant query: each variant LEFT-joined to its live mapping record, its authoritative + (measured) allele (digest / ClinGen id / VEP consequence), and the projection-group nucleotide + partner — mirroring the lean whole-set view's join + (:func:`mavedb.lib.score_set_variants.get_lean_score_set_variants`). LEFT joins with ``live_at`` in the + ON clause keep unmapped variants (with null mapped fields); a WHERE would collapse them out. The + protein slot and the gnomAD/ClinVar dimensions are resolved by separate fetches and stitched on by id. + """ + projection_link = aliased(MappingRecordAllele) + projection_allele = aliased(Allele) + query = ( + select( + Variant, + MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, + MappingRecord.hgvs_assay_level, + Allele.id.label("allele_id"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + projection_allele.level.label("projection_level"), + # Exactly one of the projection's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + # coalesce yields its canonical string regardless of which nucleotide level it sits at. + func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), + ) + .outerjoin( + projection_link, + and_( + projection_link.mapping_record_id == MappingRecord.id, + projection_link.projection_group == MappingRecordAllele.projection_group, + projection_link.is_authoritative.is_(False), + projection_link.live_at(as_of), + ), + ) + .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) + ) + return _scope_query(query, score_set_id=score_set_id, variant_ids=variant_ids) + + +def _mapped_row(row: Any, protein_hgvs_by_record: dict[int, str]) -> CsvMappedRow: + """Assemble one variant's :class:`CsvMappedRow` from a substrate-query row: the canonical post-mapped + HGVS at each level (reconstructed via the shared projection rule) plus the assay-level string, VRS id, + VEP consequence, and ClinGen id off the authoritative allele.""" + slots = mapped_hgvs_by_level( + assay_level=SequenceLevel(row.assay_level) if row.assay_level else None, + assay_level_hgvs=row.hgvs_assay_level, + projection_level=row.projection_level, + projection_hgvs=row.projection_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ) + return CsvMappedRow( + hgvs_g=slots.get(SequenceLevel.genomic.value), + hgvs_c=slots.get(SequenceLevel.cdna.value), + hgvs_p=slots.get(SequenceLevel.protein.value), + hgvs_assay_level=row.hgvs_assay_level, + vrs_digest=row.vrs_digest, + vep_functional_consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + ) + + +def _gnomad_by_allele(db: Session, allele_ids: list[int], *, as_of: Optional[datetime]) -> dict[int, GnomADVariant]: + """The live gnomAD frequency record for each of *allele_ids*, keyed by allele id. + + Liveness alone selects the release; there is deliberately no ``db_version`` predicate. The link is + allele-keyed and single-live (``uq_gnomad_allele_links_live``), and ingestion supersedes on that key, + so a release bump retires the prior link rather than accumulating one per version — whatever is live + *is* the release we hold for that allele. Filtering on the currently-served constant on top of that + was wrong in two ways: + + - The linker only visits alleles present in the release it ingests, so an allele absent from the new + release keeps its prior-release link live. Mixed versions are the steady state, not a transient, and + the constant turned every straggler's frequency into a permanent NA. + - Under ``as_of`` the constant is *today's* release, so reconstructing a past instant found the + then-live link and then discarded it — a silent dropout rather than a reconstruction. + + ``db_name`` is still constrained: that is source identity, not a version axis. The release each row + came from is reported in the ``gnomad.gnomad_version`` column, so the output stays self-describing. + """ + if not allele_ids: + return {} + + rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + .where(GnomADVariant.db_name == GNOMAD_DB_NAME) + ) + return {allele_id: gnomad_variant for allele_id, gnomad_variant in rows.tuples()} + + +def _clinvar_by_allele( + db: Session, allele_ids: list[int], clinvar_namespaces: dict[str, str], *, as_of: Optional[datetime] +) -> dict[str, dict[int, ClinvarControl]]: + """For each requested ClinVar namespace, the live control at that release for each of *allele_ids* + (``allele id -> control``). ClinVar links are multi-live (one per release), so each namespace's query + is narrowed to its own ``db_version``.""" + by_namespace: dict[str, dict[int, ClinvarControl]] = {} + for ns, db_version in clinvar_namespaces.items(): + per_allele: dict[int, ClinvarControl] = {} + if allele_ids: + rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(ClinvarControl.db_name == CLINVAR_DB_NAME, ClinvarControl.db_version == db_version) + ) + per_allele = {allele_id: control for allele_id, control in rows.tuples()} + + by_namespace[ns] = per_allele + + return by_namespace def fetch_variant_csv_data( @@ -36,18 +203,20 @@ def fetch_variant_csv_data( *, score_set: Optional[ScoreSet] = None, variant_ids: Optional[Sequence[int]] = None, - mapped_variant_ids: Optional[Sequence[int]] = None, start: Optional[int] = None, limit: Optional[int] = None, + as_of: Optional[datetime] = None, ) -> CsvFetchResult: """Fetch variant data from the database for CSV generation. Args: score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with *variant_ids*, which returns an explicit set in the order given. Exactly one is required. - mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has - already chosen one, since re-resolving on ``current`` alone could pick a different row and - emit a variant twice — nothing in the schema stops two mappings claiming to be current. + as_of: reconstruct the mapping-derived namespaces (reference HGVS, VEP, gnomAD, ClinVar) as they + stood at this instant, over the variant's immutable submitted HGVS/scores/counts. Defaults to + currently-live rows. The deterministic replacement for the old ``current``-flag pinning: a + live mapping record and its authoritative allele link are unique by construction, so there is + no race to guard against by pinning specific row ids. """ if (score_set is None) == (variant_ids is None): raise ValueError("exactly one of score_set or variant_ids must be provided") @@ -60,117 +229,70 @@ def fetch_variant_csv_data( need_gnomad = any(spec.needs_gnomad for spec in specs) need_score_set = any(spec.needs_score_set for spec in specs) - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = select(*select_columns) - - if score_set is not None: - query = query.where(Variant.score_set_id == score_set.id).order_by( - cast(func.split_part(Variant.urn, "#", 2), Integer) - ) - else: - query = query.where(Variant.id.in_(variant_ids or [])) - - if need_score_set: - query = query.options( + score_set_id = score_set.id if score_set is not None else None + score_set_options = ( + [ selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), - ) + ] + if need_score_set + else [] + ) - if need_mappings: - mapping_on_clause = ( - and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) - if mapped_variant_ids is not None - else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) - ) - query = query.join(MappedVariant, mapping_on_clause, isouter=True) - - # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to - # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( - GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + if not need_mappings: + # Fast path: scores/counts (and bare core/score-set identity) are a straight paginated scan, + # untouched by as_of (it only reconstructs the — absent here — mapping-derived namespaces). + query = _apply_pagination( + _scope_query(select(Variant), score_set_id=score_set_id, variant_ids=variant_ids).options( + *score_set_options ), - isouter=True, + start, + limit, ) + result = list(db.scalars(query).all()) + if variant_ids is not None: + result = _restore_variant_id_order(result, variant_ids, key=lambda v: v.id) + return CsvFetchResult(result, None, None, None) - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) + query = _apply_pagination( + _substrate_query(score_set_id=score_set_id, variant_ids=variant_ids, as_of=as_of).options(*score_set_options), + start, + limit, + ) + rows = list(db.execute(query).all()) + if variant_ids is not None: + rows = _restore_variant_id_order(rows, variant_ids, key=lambda row: row.Variant.id) - result = db.execute(query).all() + # Protein slot: the standalone protein subquery over the whole set, stitched back by record id. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set_id, variant_ids=variant_ids, as_of=as_of) + # The authoritative allele ids on this page — the join key for the batch gnomAD/ClinVar fetches. + allele_ids = [row.allele_id for row in rows if row.allele_id is not None] + gnomad_by_allele = _gnomad_by_allele(db, allele_ids, as_of=as_of) if need_gnomad else {} + clinvar_by_allele = _clinvar_by_allele(db, allele_ids, clinvar_namespaces, as_of=as_of) - # Postgres does not preserve IN-list order, so restore the caller's ordering. - if variant_ids is not None: - position = {variant_id: index for index, variant_id in enumerate(variant_ids)} - result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data_list is not None: - idx = 2 if need_mappings else 1 - gnomad_data_list.append(row[idx]) - - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - - # One query per namespace, since each names a different release; keyed by MappedVariant id and - # projected back onto row order below. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == CLINVAR_DB_NAME, - aliased_cc.db_version == db_version, - ) - ) - ) - - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - - clinvar_data_map[ns] = mv_to_cc - - clinvar_per_variant = [ - { - ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None - for ns, mv_to_cc in clinvar_data_map.items() - } - for mapping in mappings - ] + variants: list[Variant] = [] + mappings: list[Optional[CsvMappedRow]] = [] + gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = ( + [] if clinvar_namespaces else None + ) + for row in rows: + variants.append(row.Variant) + allele_id = row.allele_id + mappings.append(_mapped_row(row, protein_hgvs_by_record) if row.mapping_record_id is not None else None) + if gnomad_data is not None: + gnomad_data.append(gnomad_by_allele.get(allele_id) if allele_id is not None else None) + if clinvar_per_variant is not None: + clinvar_per_variant.append( + { + ns: (clinvar_by_allele[ns].get(allele_id) if allele_id is not None else None) + for ns in clinvar_namespaces + } + ) return CsvFetchResult( variants=variants, mappings=mappings, - gnomad_data=gnomad_data_list, + gnomad_data=gnomad_data, clinvar_per_variant=clinvar_per_variant, ) diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index bbaf33c13..f79f05983 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -1,5 +1,6 @@ """The score-set CSV export: every variant in one score set, and the columns it can offer.""" +from datetime import datetime from typing import List, Optional from sqlalchemy import and_, select @@ -38,6 +39,7 @@ def get_score_set_variants_as_csv( start: Optional[int] = None, limit: Optional[int] = None, drop_unused_hgvs_columns_flag: Optional[bool] = None, + as_of: Optional[datetime] = None, viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" @@ -53,6 +55,7 @@ def get_score_set_variants_as_csv( score_set=score_set, start=start, limit=limit, + as_of=as_of, ) mappings = fetched.mappings or [None] * len(fetched.variants) @@ -78,12 +81,19 @@ def available_score_set_csv_namespaces( db: Session, score_set: ScoreSet, viewer: Optional[ScoreCalibrationViewer] = None, + *, + as_of: Optional[datetime] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. Its own endpoint rather than a field on the score-set response: it costs several queries and is only needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; it just produces a column of NA. + + ``as_of`` must match the instant the caller will download at. Discovery pinned to "now" against an + ``as_of`` download disagrees in both directions: it offers mapping namespaces that come back entirely + NA, and omits ones the download could have filled. The ClinVar case is the sharpest, since the live + release set decides which ``clinvar.YYYY_MM`` headers exist at all. """ dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} # TODO(#372): non-null id fields @@ -101,12 +111,12 @@ def available_score_set_csv_namespaces( entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance - if score_sets_have_current_mappings(db, score_set_ids): + if score_sets_have_current_mappings(db, score_set_ids, as_of=as_of): entries.extend( static_namespace_entry(ns) for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) ) - entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids, as_of=as_of))) # Every calibration the score set defines is offered, rangeless ones included. calibrations = db.scalars( diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index 99484cdcf..bd37aa240 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -8,10 +8,26 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN from mavedb.lib.validation.constants.general import hgvs_nt_column, hgvs_pro_column, hgvs_splice_column -from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant + +@dataclass(frozen=True) +class CsvMappedRow: + """The mapping-derived CSV fields for one variant, resolved from its live mapping record's + authoritative allele (post-mapped HGVS per level, assay-level HGVS, VRS digest, VEP consequence, + ClinGen id). HGVS strings are already resolved to their canonical per-level form by the fetch layer + (see ``mavedb.lib.score_set_variants.mapped_hgvs_by_level``) — no raw VRS payload to fall back on + parsing here, unlike the pre-allele-graph ``MappedVariant`` substrate.""" + + hgvs_g: Optional[str] + hgvs_c: Optional[str] + hgvs_p: Optional[str] + hgvs_assay_level: Optional[str] + vrs_digest: Optional[str] + vep_functional_consequence: Optional[str] + clingen_allele_id: Optional[str] + + # One entry per namespace, so adding one is a single edit. This previously took three unrelated changes # — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. @@ -110,55 +126,6 @@ def resolver(self, column_key: str) -> Optional[Callable]: return _optional(lambda data: data.get(column_key)) -def _safe_hgvs_from_post_mapped(mapping: MappedVariant) -> Optional[str]: - """``get_hgvs_from_post_mapped`` with its raises absorbed, since a cell cannot afford to raise. - - ``hgvs_from_vrs_allele`` raises on two payload shapes: a VRS 1.x object, which it refuses - deliberately, and an allele with no ``expressions`` key. Raised from inside a cell resolver, either - would abort the export of every other row in the file. Returning None keeps the refusal — no HGVS is - emitted for those payloads — without letting one variant's shape cost the caller the whole download. - - Intentionally scoped to the CSV export layer, where a single problematic variant should not abort the entire export. - """ - if not mapping.post_mapped: - return None - try: - return get_hgvs_from_post_mapped(mapping.post_mapped) - except (KeyError, ValueError): - return None - - -def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: - """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" - if mapping is None: - return None - if mapping.hgvs_g: - return str(mapping.hgvs_g) - fallback = _safe_hgvs_from_post_mapped(mapping) - return fallback if fallback is not None and is_hgvs_g(fallback) else None - - -def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: - """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" - if mapping is None: - return None - if mapping.hgvs_p: - return str(mapping.hgvs_p) - fallback = _safe_hgvs_from_post_mapped(mapping) - return fallback if fallback is not None and is_hgvs_p(fallback) else None - - -def _post_mapped_vrs_id(mapping: Optional[MappedVariant]) -> Optional[str]: - """The GA4GH identifier of the post-mapped VRS object, or None if there is no post-mapped object. - - The full ``ga4gh:VA.{digest}`` identifier rather than the bare digest, so a value copied out of an - export can be pasted straight into the VRS search, which validates against the GA4GH CURIE form. - """ - if mapping is None or not mapping.post_mapped: - return None - return get_id_from_post_mapped(mapping.post_mapped) - - def _target_genes(variant: Variant) -> Optional[str]: """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" if not variant.score_set: @@ -193,16 +160,14 @@ def _optional(getter: Callable) -> Callable: emit_under=CsvNamespace.SCORES, ), CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), - # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. - # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( source=RowSource.MAPPING, resolvers={ - "post_mapped_hgvs_g": _post_mapped_hgvs_g, - "post_mapped_hgvs_p": _post_mapped_hgvs_p, - "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), - "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), - "post_mapped_vrs_id": _post_mapped_vrs_id, + "post_mapped_hgvs_g": _optional(attrgetter("hgvs_g")), + "post_mapped_hgvs_p": _optional(attrgetter("hgvs_p")), + "post_mapped_hgvs_c": _optional(attrgetter("hgvs_c")), + "post_mapped_hgvs_at_assay_level": _optional(attrgetter("hgvs_assay_level")), + "post_mapped_vrs_id": _optional(attrgetter("vrs_digest")), }, needs_mappings=True, ), diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py index 61e657e76..932abea74 100644 --- a/src/mavedb/lib/csv/variant.py +++ b/src/mavedb/lib/csv/variant.py @@ -9,6 +9,7 @@ """ import logging +from datetime import datetime from typing import Any, Callable, Optional from sqlalchemy import and_, select @@ -40,7 +41,9 @@ from mavedb.lib.mave.utils import NA_VALUE from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet @@ -88,13 +91,15 @@ def _equivalent_measurements( db: Session, variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, -) -> Optional[list[tuple[int, int, int]]]: + *, + as_of: Optional[datetime] = None, +) -> Optional[list[tuple[int, int]]]: """Resolve a variant URN to the measurements the CSV should report. - Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every - other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated - downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer - restore this order from variant ids alone. + Returns ``[(variant_id, score_set_id), ...]``, requested variant first, then every other measurement + live as of *as_of* against the same ClinGen allele, ordered by score set and variant URN so repeated + downloads are byte-identical. A live mapping record and its authoritative allele link are unique by + construction, so at most one entry exists per variant without needing to pin or deduplicate. Args: may_read_score_set: when given, drops measurements from score sets the caller may not read. @@ -102,88 +107,93 @@ def _equivalent_measurements( requested variant is not repeated here. Returns: - None when the variant has no current mapping, since there is then no allele to expand by. + None when the variant has no live mapping as of *as_of*, since there is then no allele to expand by. """ - # Nothing in the schema enforces one current mapping per variant, and every column below follows from - # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. - requested = db.scalars( - select(MappedVariant) - .join(MappedVariant.variant) - .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) - .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + requested = db.execute( + select(Variant.id, Variant.score_set_id, Allele.clingen_allele_id) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(Variant.urn == variant_urn) .limit(1) ).one_or_none() if requested is None: return None + requested_variant_id, requested_score_set_id, clingen_allele_id = requested + # TODO(#372): non-null id fields - if not requested.clingen_allele_id: - return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + if not clingen_allele_id: + return [(requested_variant_id, requested_score_set_id)] # type: ignore - # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the - # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through - # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. equivalents = db.execute( - select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) - .join(MappedVariant.variant) + select(Variant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) .join(Variant.score_set) .where( and_( - MappedVariant.clingen_allele_id == requested.clingen_allele_id, - MappedVariant.current.is_(True), - # By variant, not by mapping: the anchor already represents this variant, and a second - # current mapping on it is the same measurement again, not an equivalent one. - MappedVariant.variant_id != requested.variant_id, + Allele.clingen_allele_id == clingen_allele_id, + # By variant, not by mapping: the anchor already represents this variant, and its own + # live mapping is the same measurement again, not an equivalent one. + Variant.id != requested_variant_id, ) ) - .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) ).all() - # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order - # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. - deduplicated: dict[int, Any] = {} - for row in equivalents: - deduplicated.setdefault(row[0], row) - equivalents = list(deduplicated.values()) - if may_read_score_set is not None and equivalents: - candidate_score_set_ids = {row[2] for row in equivalents} + candidate_score_set_ids = {row[1] for row in equivalents} readable_score_set_ids = { score_set.id for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() if may_read_score_set(score_set) } - equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + equivalents = [row for row in equivalents if row[1] in readable_score_set_ids] equivalents = sorted( equivalents, - key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + key=lambda row: (score_set_urn_sort_key(row[2]), variant_urn_sort_key(row[3])), ) - return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ - (row[0], row[1], row[2]) for row in equivalents - ] + return [(requested_variant_id, requested_score_set_id)] + [(row[0], row[1]) for row in equivalents] -def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: - """The namespaces to offer for a variant that exists but has no current mapping. +def _unmapped_variant_namespaces(db: Session, score_set_id: int, *, as_of: Optional[datetime] = None) -> list[str]: + """The namespaces to offer for a variant that exists but has no mapping live as of *as_of*. Always score, score set, and relationship; plus the mapping-derived groups when the score set has been mapped at all, where NA is the honest value for a variant the mapper has not reached. """ - if score_sets_have_current_mappings(db, [score_set_id]): + if score_sets_have_current_mappings(db, [score_set_id], as_of=as_of): return list(BASE_VARIANT_CSV_NAMESPACES) return list(ALWAYS_AVAILABLE_NAMESPACES) -def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: +def _latest_clinvar_namespace( + db: Session, score_set_ids: list[int], *, as_of: Optional[datetime] = None +) -> Optional[str]: """The ClinVar namespace for the most recent release covering these measurements' score sets. One release for the whole file rather than each variant's own latest: that keeps the release in the column header where it is citable, and avoids mixing calls from different releases in one column. """ - namespaces = clinvar_release_namespaces(db, score_set_ids) + namespaces = clinvar_release_namespaces(db, score_set_ids, as_of=as_of) if not namespaces: return None @@ -236,6 +246,8 @@ def available_variant_csv_namespaces( variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, viewer: Optional[ScoreCalibrationViewer] = None, + *, + as_of: Optional[datetime] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. @@ -245,7 +257,7 @@ def available_variant_csv_namespaces( Raises: ValueError: if no variant with *variant_urn* exists. """ - measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set, as_of=as_of) if measurements is None: variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() @@ -255,16 +267,19 @@ def available_variant_csv_namespaces( # No mapping on this variant, but its score set may still be mapped, in which case the # mapping-derived columns are owed with NA rather than omitted. # TODO(#372): non-null id fields - return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + return [ + static_namespace_entry(ns) + for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id), as_of=as_of) # type: ignore + ] base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] - score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + score_set_ids = list({score_set_id for _, score_set_id in measurements}) return ( base_entries + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) - + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids, as_of=as_of)) ) @@ -275,33 +290,36 @@ def get_variant_csv( may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, viewer: Optional[ScoreCalibrationViewer] = None, na_rep: str = NA_VALUE, + *, + as_of: Optional[datetime] = None, ) -> str: """Build the clinical CSV for a variant and its equivalent measurements. - One row per measurement: the requested variant first, then every other current measurement of the - same ClinGen allele across score sets. + One row per measurement: the requested variant first, then every other measurement live as of + *as_of* against the same ClinGen allele, across score sets. Args: namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar release covering these measurements. + as_of: reconstruct every measurement (and its mapping-derived columns) as it stood at this + instant. Defaults to currently-live rows. Raises: ValueError: if no variant with *variant_urn* exists. """ - measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set, as_of=as_of) if measurements is None: - return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep, as_of=as_of) - variant_ids = [variant_id for variant_id, _, _ in measurements] - mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] - score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + variant_ids = [variant_id for variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, score_set_id in measurements}) calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) if namespaces is None: - clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids, as_of=as_of) # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. default_calibrations = sorted( namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only @@ -321,7 +339,7 @@ def get_variant_csv( columns, plan.clinvar_namespaces, variant_ids=variant_ids, - mapped_variant_ids=mapped_variant_ids, + as_of=as_of, ) mappings = fetched.mappings or [None] * len(fetched.variants) @@ -346,8 +364,10 @@ def _unmapped_variant_csv( variant_urn: str, namespaces: Optional[list[str]] = None, na_rep: str = NA_VALUE, + *, + as_of: Optional[datetime] = None, ) -> str: - """Build the single-row CSV for a variant that exists but has no current mapping. + """Build the single-row CSV for a variant that exists but has no mapping live as of *as_of*. Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the @@ -368,7 +388,9 @@ def _unmapped_variant_csv( dataset_columns={}, namespaces=( # TODO(#372): non-null id fields - list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + list(namespaces) + if namespaces is not None + else _unmapped_variant_namespaces(db, int(variant.score_set_id), as_of=as_of) # type: ignore ), ) columns = plan.namespaced_columns diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9bfa0fec9..830cc62d9 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,24 +1,115 @@ import logging import os import re -from typing import Any, Sequence, Union +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Any, Optional, Sequence, Union -from sqlalchemy import Connection, Row, select, text +from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session -from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import batched -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant -GNOMAD_DB_NAME = "gnomAD" -GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") # e.g., "v4.1" logger = logging.getLogger(__name__) +GNOMAD_DB_NAME = "gnomAD" +GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") +_CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" +""" +Strip leading zeros from a CAID's numeric portion, keeping at least one digit. +Kept byte-for-byte in sync with the SQL form used to normalize Allele.clingen_allele_id +in link_gnomad_variants_to_alleles. +""" + + +class GnomadLinkVerdict(str, Enum): + """Per-allele outcome of a gnomAD linking run, returned for every allele the linker touched. + + The single source of truth for what happened to an allele's link this run — the caller derives + annotation status directly from this, never by re-querying link state (which would be a second, + drift-prone source of truth). + """ + + CREATED = "created" # link created or superseded this run (a new/changed live link) + UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched + + +@dataclass(frozen=True) +class GnomadVariantLink: + """One (score-set variant, annotated allele) pair a gnomAD frequency record reaches. + + Mirrors :class:`mavedb.lib.clinical_controls.ControlVariantLink`. ``allele_digest`` is the VRS digest + of the allele gnomAD's frequency was linked to. + """ + + variant_urn: str + allele_digest: str + + +def get_gnomad_variants_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_version: Optional[str] = None, +) -> list[tuple[GnomADVariant, list[GnomadVariantLink]]]: + """The live gnomAD frequency records for a score set, each paired with the score-set variants that link + to it. + + Walks ``GnomadAlleleLink → Allele → MappingRecordAllele → MappingRecord → Variant`` with every + ``ValidTime`` hop constrained to the same instant (``as_of``, defaulting to currently-live). Optionally + narrowed to one gnomAD release (``db_version``). Records come back in first-seen order; each + record's links preserve query order. + """ + stmt = ( + select( + GnomADVariant, + Variant.urn.label("variant_urn"), + Allele.vrs_digest.label("allele_digest"), + ) + .join(GnomadAlleleLink, GnomadAlleleLink.gnomad_variant_id == GnomADVariant.id) + .join(Allele, Allele.id == GnomadAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(GnomadAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + if db_version is not None: + stmt = stmt.where(GnomADVariant.db_version == db_version) + + variants: dict[int, GnomADVariant] = {} + links_by_variant: dict[int, list[GnomadVariantLink]] = {} + + for gnomad_variant, variant_urn, allele_digest in db.execute(stmt).tuples(): + # TODO#372: nullable URN + if variant_urn is None: + continue + + if gnomad_variant.id not in variants: + variants[gnomad_variant.id] = gnomad_variant + links_by_variant[gnomad_variant.id] = [] + + links_by_variant[gnomad_variant.id].append( + GnomadVariantLink(variant_urn=variant_urn, allele_digest=allele_digest) + ) + + return [(gnomad_variant, links_by_variant[gnomad_variant.id]) for gnomad_variant in variants.values()] + + def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: """ Generate a gnomAD variant identifier based on contig, position, and alleles. @@ -46,6 +137,18 @@ def gnomad_table_name() -> str: return table_name +def normalize_caid(caid: str) -> str: + """Normalize a ClinGen CAID by stripping leading zeros from its numeric portion. + + The gnomAD Hail/Athena dump drops leading zeros from CAIDs — MaveDB's ``CA025094`` is recorded as + ``CA25094`` — so an exact-string join silently misses every zero-padded CAID (issue #722). Both + sides of the join are normalized to the unpadded form to repair the match. ``CA025094`` and + ``CA25094`` denote the same ClinGen allele, so this can never collide distinct alleles. A value + that is not a recognizable CAID (no ``CA`` prefix + digits) is returned unchanged. + """ + return re.sub(_CAID_LEADING_ZERO_RE, r"\1\2", caid) + + def allele_list_from_list_like_string(alleles_string: str) -> list[str]: """ Convert a list-like string representation of alleles into a Python list. @@ -94,8 +197,9 @@ def gnomad_variant_data_for_caids( Raises: sqlalchemy.exc.SQLAlchemyError: If there is an error executing the query. """ + # Normalize to the unpadded form the dump stores so the IN-list matches zero-padded CAIDs (see issue #722). chunked_caids = batched(caids, 16250) - caid_strs = [",".join(f"'{caid}'" for caid in chunk) for chunk in chunked_caids] + caid_strs = [",".join(f"'{normalize_caid(caid)}'" for caid in chunk) for chunk in chunked_caids] save_to_logging_context({"num_caids": len(caids), "num_chunks": len(caid_strs)}) result_rows: list[Row[Any]] = [] @@ -132,31 +236,52 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_mapped_variants( - db: Session, gnomad_variant_data: Sequence[Row[Any]], only_current: bool = True -) -> int: - """ - Links gnomAD variants to mapped variants in the database based on CAIDs. Note that this function does - not commit this data to the database; it only prepares the relationships. - - Args: - caids (list[str]): A list of CAIDs to link with gnomAD variants. +def link_gnomad_variants_to_alleles( + db: Session, gnomad_variant_data: Sequence[Row[Any]] +) -> dict[int, GnomadLinkVerdict]: + """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. + + Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a + valid-time :class:`GnomadAlleleLink`, so one gnomAD variant fans out to every allele sharing the + CAID (cross-score-set dedup included). Each allele holds at most one live link, superseded **only + on change**: a live link already pointing to the resolved variant is left untouched (an unchanged + re-run writes no spurious valid-time boundary); a new/different/older-version target retires it and + inserts a successor, keyed on ``allele_id`` so a version bump replaces rather than accumulates. The + guard is load-bearing despite the job's upstream skip — shared CAIDs and ``force`` runs still reach + it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is + logged and superseded newest-wins, not raised. + + Does not commit. Returns a verdict per allele *touched* this run (matched a CAID-bearing row): + :attr:`GnomadLinkVerdict.CREATED` for a created/superseded link, :attr:`~GnomadLinkVerdict.UNCHANGED` + for a live link left in place. Alleles absent from the map were matched by no row — the caller reads + those as "gnomAD had no record". This is the single source of truth for per-allele status; callers + must not re-derive it by re-querying link state. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) - save_to_logging_context({"only_current": only_current}) - logger.debug(msg="Linking gnomAD variants to mapped variants", extra=logging_context()) + logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) + + # Resolve all rows' alleles in one query, then group in Python. The regexp_replace predicate + # (normalizing the zero-padded stored CAID to the dump's stripped form, #722) is non-sargable, so + # a per-row query would seq-scan `alleles` once per variant; one IN scan replaces those N scans. + target_caids = {normalize_caid(row.caid) for row in gnomad_variant_data} + alleles_by_caid: dict[str, list[Allele]] = defaultdict(list) + if target_caids: + for allele in db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2").in_(target_caids) + ) + ).all(): + alleles_by_caid[normalize_caid(allele.clingen_allele_id)].append(allele) - linked_gnomad_variants = 0 - annotation_manager = AnnotationStatusManager(db) + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - mapped_variants_with_caids_query = select(MappedVariant).where(MappedVariant.clingen_allele_id == row.caid) - if only_current: - mapped_variants_with_caids_query = mapped_variants_with_caids_query.where(MappedVariant.current.is_(True)) - mapped_variants_with_caids = db.scalars(mapped_variants_with_caids_query).all() + alleles_with_caid = alleles_by_caid.get(normalize_caid(row.caid), []) + if not alleles_with_caid: + continue gnomad_identifier_for_variant = gnomad_identifier( row.__getattribute__("locus.contig"), @@ -172,80 +297,88 @@ def link_gnomad_variants_to_mapped_variants( if faf95_max is not None: faf95_max = float(faf95_max) - for mapped_variant in mapped_variants_with_caids: - # Remove any existing gnomAD variants for this mapped variant that match the current gnomAD data version to avoid data duplication. - # There should only be one gnomAD variant per mapped variant per gnomAD data version, since each gnomAD variant can only match to one - # CAID. - for linked_gnomad_variant in mapped_variant.gnomad_variants: - if linked_gnomad_variant.db_version == GNOMAD_DATA_VERSION: - logger.debug( - msg=f"Removing existing gnomAD variant {linked_gnomad_variant.db_identifier} from mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", - extra=logging_context(), - ) - mapped_variant.gnomad_variants.remove(linked_gnomad_variant) - - existing_gnomad_variant = db.scalar( - select(GnomADVariant).where( - GnomADVariant.db_name == "gnomAD", - GnomADVariant.db_identifier == gnomad_identifier_for_variant, - GnomADVariant.db_version == GNOMAD_DATA_VERSION, - ) + # One gnomAD variant per (identifier, version): get-or-create so repeated CAIDs and re-runs + # reuse the same row. Flush so a freshly created variant has an id for the link below. + gnomad_variant = db.scalar( + select(GnomADVariant).where( + GnomADVariant.db_name == GNOMAD_DB_NAME, + GnomADVariant.db_identifier == gnomad_identifier_for_variant, + GnomADVariant.db_version == GNOMAD_DATA_VERSION, + ) + ) + if gnomad_variant is None: + logger.debug( + msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), + ) + gnomad_variant = GnomADVariant( + db_name=GNOMAD_DB_NAME, + db_identifier=gnomad_identifier_for_variant, + db_version=GNOMAD_DATA_VERSION, + allele_count=allele_count, + allele_number=allele_number, + allele_frequency=allele_frequency, # type: ignore + faf95_max_ancestry=faf95_max_ancestry, + faf95_max=faf95_max, # type: ignore + ) + db.add(gnomad_variant) + db.flush() + else: + logger.debug( + msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), ) - if existing_gnomad_variant is None: - logger.debug( - msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", - extra=logging_context(), + for allele in alleles_with_caid: + live_link = db.scalar( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, ) - gnomad_variant = GnomADVariant( - db_name=GNOMAD_DB_NAME, - db_identifier=gnomad_identifier_for_variant, - db_version=GNOMAD_DATA_VERSION, - allele_count=allele_count, - allele_number=allele_number, - allele_frequency=allele_frequency, # type: ignore - faf95_max_ancestry=faf95_max_ancestry, - faf95_max=faf95_max, # type: ignore - ) - else: - logger.debug( - msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + ) + + # No change: live link already points here — leave it untouched (no spurious boundary). + if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + verdicts.setdefault(allele.id, GnomadLinkVerdict.UNCHANGED) + continue + + if ( + live_link is not None + and live_link.gnomad_variant.db_version == GNOMAD_DATA_VERSION + and live_link.gnomad_variant.db_identifier != gnomad_identifier_for_variant + ): + logger.warning( + msg=( + f"CAID {allele.clingen_allele_id} for allele {allele.id} resolved to " + f"{gnomad_identifier_for_variant} at version {GNOMAD_DATA_VERSION}, but a live link " + f"already points to {live_link.gnomad_variant.db_identifier} at the same version. " + "Superseding (newest wins); investigate the gnomAD source for a re-resolved CAID." + ), extra=logging_context(), ) - gnomad_variant = existing_gnomad_variant - if gnomad_variant not in mapped_variant.gnomad_variants: - mapped_variant.gnomad_variants.append(gnomad_variant) - linked_gnomad_variants += 1 - - db.add(gnomad_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "gnomad_db_identifier": gnomad_variant.db_identifier, - } - }, - current=True, + # Change: retire any live link for the allele, insert the successor (allele-keyed, so a + # version bump replaces rather than accumulates). + GnomadAlleleLink.supersede_live_where( + db, + [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], + GnomadAlleleLink.allele_id == allele.id, ) + verdicts[allele.id] = GnomadLinkVerdict.CREATED # created always wins over a same-run unchanged logger.debug( - msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", + msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", extra=logging_context(), ) logger.info( - f"Linked {len(mapped_variants_with_caids)} mapped variants with CAID {row.caid} to gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" + f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - annotation_manager.flush() - - save_to_logging_context({"linked_gnomad_variants": linked_gnomad_variants}) + changed_allele_count = sum(1 for v in verdicts.values() if v is GnomadLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Linked a total of {linked_gnomad_variants} gnomAD variants to mapped variants.", + msg=f"Created or superseded {changed_allele_count} allele links this run.", extra=logging_context(), ) - return linked_gnomad_variants + return verdicts diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py new file mode 100644 index 000000000..f974506b9 --- /dev/null +++ b/src/mavedb/lib/hgvs.py @@ -0,0 +1,182 @@ +import re +import sys +from dataclasses import dataclass +from typing import Optional + +# Coordinate prefix of an HGVS variant description: a single type letter plus a dot +# (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. +_HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") + +_FIRST_INT = re.compile(r"\d+") + + +def _cis_phased_sort_key(description: str) -> tuple[int, str]: + """Order key for a cis-phased component description by its first integer position. + + The first run of digits is the coordinate for both genomic (``123A>G``) and protein + (``Arg123Gly``) forms. Descriptions with no digit sort last, and the raw string breaks ties so + the order is total and stable. + """ + match = _FIRST_INT.search(description) + return (int(match.group()) if match else sys.maxsize, description) + + +def extract_accession(hgvs_string: str) -> str: + """Extract the reference accession from an HGVS string, or return empty string if it cannot + be parsed. + + This function makes no assumptions about the structure of the accession (e.g. whether it starts with "NM_" + or "NC_") and is robust to extra whitespace. It simply returns the substring before the first colon, which + is the standard HGVS separator between the accession and the variant description. Callers must validate + the accession format separately if needed. + """ + token = (hgvs_string or "").strip() + if ":" not in token: + return "" + + return token.split(":", 1)[0].strip() + + +_HGVS_P_PREDICTION = re.compile(r":p\.\((.+)\)\s*$") + + +def strip_protein_prediction_parens(hgvs_p: str) -> str: + """Unwrap the prediction parentheses from a protein HGVS: ``p.(Ala222Val)`` -> ``p.Ala222Val``. + + Forward translation (``c_to_p``) emits a *predicted* consequence in parentheses. ga4gh + translates either form fine, but the parens are noise we don't want in the stored HGVS -- + they denote inference, not a different variant -- so we normalize to the bare form for + consistency. A string with no prediction parens is returned unchanged. + """ + return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) + + +_NT_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?[cgn]\.(\d+)([ACGTacgt])>([ACGTacgt])$") +""" +Nucleotide: only a purely numeric position yields a placeable block — UTR/intron coding positions +(`c.*123`, `c.-12`, `c.12+3`) are not single integers and so are not parsed into a block (the +heatmap cannot place them either). ref/alt are single nucleotides. +""" +_PRO_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?p\.\(?([A-Za-z]{3})(\d+)([A-Za-z]{3}|=|\*|-)\)?$") +""" +Protein: three-letter ref, integer position, and a three-letter alt or one of `=` (synonymous), +`*` (Ter/stop), `-` (deletion) — matching MaveHGVS-pro and the heatmap's amino-acid rows. +""" + + +@dataclass(frozen=True) +class SequenceBlock: + """A parsed single-locus substitution: 1-based ``position`` plus ``ref`` and ``alt`` residues. + + For coding DNA ``ref``/``alt`` are single nucleotides; for protein they are three-letter amino-acid + codes or one of ``=`` (synonymous), ``*`` (Ter/stop), ``-`` (deletion). + """ + + position: int + ref: str + alt: str + + +def parse_simple_nucleotide_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus coding/genomic substitution (``c.1216G>A``) into a :class:`SequenceBlock`. + + Returns ``None`` for anything that is not a numeric-position single-nucleotide substitution + (multivariant, indel, UTR/intron position, empty), so non-placeable variants simply yield no block. + """ + if not hgvs: + return None + match = _NT_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(1)), ref=match.group(2), alt=match.group(3)) + + +def parse_simple_protein_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus protein substitution (``p.Ala406Thr``, ``p.(Ala406Thr)``) into a + :class:`SequenceBlock`. + + The alt is the raw MaveHGVS token: a three-letter code, ``=`` (synonymous), ``*`` (Ter), or ``-`` + (deletion). Returns ``None`` for anything else (multivariant, frameshift, empty). + """ + if not hgvs: + return None + match = _PRO_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(2)), ref=match.group(1), alt=match.group(3)) + + +def parse_simple_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus nucleotide *or* protein substitution, auto-detecting by coordinate prefix. + + The nucleotide (``c.``/``g.``/``n.``) and protein (``p.``) forms are disjoint, so trying both is + unambiguous — useful when the level is not known up front (e.g. the assay-level HGVS, which is + coding for a nucleotide assay and protein for a protein assay). Returns ``None`` for anything that + is not a placeable simple substitution. + """ + return parse_simple_nucleotide_substitution(hgvs) or parse_simple_protein_substitution(hgvs) + + +def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: + """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. + + The reverse-translate-variants tool emits the non-adjacent component substitutions of a + single codon change as one bracketed genomic expression (``NC_000001.11:g.[123A>G;125T>C]``) + whenever they are too far apart on the genome to collapse into a single delins. ga4gh's + AlleleTranslator only ever produces a single Allele, so each component must be translated + independently and recombined into a VRS CisPhasedBlock. + + Each returned component carries the original accession and coordinate prefix + (``NC_000001.11:g.123A>G``). A non-bracketed expression is returned unchanged as a + single-element list, so callers can treat both cases uniformly. + + Unlike a bare mavehgvs split, the accession is preserved: the components feed straight + into VRS translation, which requires a reference accession to resolve positions. + """ + accession, separator, remainder = hgvs_string.partition(":") + # Only an accession-qualified, bracketed expression is a cis-phased multivariant we split here; + # anything else (bare, unbracketed, or accession-less) is returned unchanged so the caller can + # treat both cases uniformly without a ValueError on the missing ":" / "[". + if not separator or "[" not in remainder: + return [hgvs_string] + + prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." + inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] + return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] + + +def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: + """Join cis-phased component HGVS strings into one bracketed expression. + + Inverse of :func:`split_cis_phased_hgvs`: ``["NC_…:g.123A>G", "NC_…:g.125T>C"]`` becomes + ``"NC_…:g.[123A>G;125T>C]"``. A single component is returned unchanged. + + Returns ``None`` when the components do not share a single accession and coordinate prefix — + they are then not expressible as one cis-phased block (e.g. members on different sequences). + """ + if not components: + return None + if len(components) == 1: + return components[0] + + accessions: set[str] = set() + prefixes: set[str] = set() + descriptions: list[str] = [] + for component in components: + accession, separator, remainder = component.partition(":") + match = _HGVS_COORD_PREFIX.match(remainder) + if not separator or not match: + return None + + accessions.add(accession) + prefixes.add(match.group(1)) + descriptions.append(match.group(2)) + + if len(accessions) != 1 or len(prefixes) != 1: + return None + + # Emit components in coordinate order so the combined string is deterministic regardless of + # member ordering. The VRS block digest is order-independent (so dedup is unaffected), but this + # string is surfaced in CSV export, where a stable, spec-conventional ordering is useful. + descriptions.sort(key=_cis_phased_sort_key) + return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/src/mavedb/lib/mapping/schema.py b/src/mavedb/lib/mapping/schema.py index 939b60401..02d377780 100644 --- a/src/mavedb/lib/mapping/schema.py +++ b/src/mavedb/lib/mapping/schema.py @@ -7,9 +7,31 @@ """ from datetime import datetime +from enum import Enum from typing import Any, Optional, TypedDict +class MappingOutcome(str, Enum): + """Per-record outcome stamped by dcd-mapping on every emitted score annotation. + + Mirrors ``dcd_mapping.schemas.MappingOutcome``. Lets a benign absence of a VRS allele + be told apart from a genuine failure -- a distinction ``error_message`` alone cannot + make (benign outcomes leave it ``None``). ``MAPPED`` is a success; ``INTRONIC`` and + ``NO_PROTEIN_CONSEQUENCE`` are benign skips (no allele, not failures); ``FAILED`` is the + only genuine failure. + """ + + MAPPED = "mapped" + INTRONIC = "intronic" + NO_PROTEIN_CONSEQUENCE = "no_protein_consequence" + FAILED = "failed" + + @property + def is_benign_absence(self) -> bool: + """True for outcomes that legitimately produce no allele yet are not failures.""" + return self in (MappingOutcome.INTRONIC, MappingOutcome.NO_PROTEIN_CONSEQUENCE) + + class GeneInfo(TypedDict, total=False): hgnc_symbol: Optional[str] selection_method: Optional[str] diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 11e1b2e88..25608df7c 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -17,6 +17,8 @@ hgvs_pro_column, ) from mavedb.lib.validation.utilities import inf_or_float +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.enums.functional_classification import FunctionalClassification from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -27,6 +29,56 @@ from mavedb.view_models import score_calibration +def calibration_preference_key(calibration: ScoreCalibration) -> tuple[int, int, int]: + """The UI's default-calibration preference cascade as a sort key (lower sorts first = more preferred): + ``primary``, then ``investigator_provided``, then non-``research_use_only``. Mirrors the UI's + ``activeCalibrationOptions`` default selection, so the API and UI agree on which calibration is "the + default" for a score set. + + Intentionally stops at the cascade tiers and carries **no** evidence strength (a variant-specific + concern) and **no** id — callers append their own deterministic tiebreak (calibration id or urn), and, where + a within-tier strongest-evidence tiebreak is wanted, :func:`classification_evidence_strength`. + """ + return ( + 0 if calibration.primary else 1, + 0 if calibration.investigator_provided else 1, + 0 if not calibration.research_use_only else 1, + ) + + +def classification_evidence_strength( + classification: Optional[ScoreCalibrationFunctionalClassification], +) -> tuple[float, int]: + """``(magnitude, direction)`` of a functional classification's evidence, for ranking. + + ``magnitude`` is the evidence strength (larger = stronger); ``direction`` biases toward pathogenic + (0 = pathogenic, 1 = benign, 2 = unknown). ACMG points are the primary signal (signed: + positive = pathogenic), falling back to ``|ln(oddspaths_ratio)|`` then the functional call + (abnormal → pathogenic, normal → benign) for direction only. + """ + if classification is None: + return (0.0, 2) + + acmg = classification.acmg_classification + if acmg is not None and acmg.points is not None: + points = float(acmg.points) + return (abs(points), 0 if points > 0 else 1 if points < 0 else 2) + + # OddsPath is a ratio centered at 1 (pathogenic >1, benign <1); ln maps it to a symmetric log-odds + # strength around 0 (x and 1/x are equal-and-opposite), comparable across direction and commensurate + # with the already-log-scaled ACMG points. + ratio = classification.oddspaths_ratio + if ratio is not None and ratio > 0: + return (abs(math.log(ratio)), 0 if ratio > 1 else 1 if ratio < 1 else 2) + + if classification.functional_classification == FunctionalClassification.abnormal: + return (0.0, 0) + elif classification.functional_classification == FunctionalClassification.normal: + return (0.0, 1) + + return (0.0, 2) + + def create_functional_classification( db: Session, functional_range_create: Union[ @@ -773,18 +825,8 @@ def variants_for_functional_classification( continue elif functional_classification.range is not None and len(functional_classification.range) == 2: - try: - container = v.data.get("score_data") if isinstance(v.data, dict) else None - if not container or not isinstance(container, dict): - continue - - raw = container.get("score") - if raw is None: - continue - - score = float(raw) - - except Exception: # noqa: BLE001 + score = score_from_variant_data(v.data) + if score is None: continue if functional_classification.score_is_contained_in_range(score): diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py new file mode 100644 index 000000000..262d11654 --- /dev/null +++ b/src/mavedb/lib/score_set_variants.py @@ -0,0 +1,319 @@ +"""Lean whole-set view backing the score-set page (table, heatmap, score/effect histograms). + +Assembles one record per variant in two O(N) bulk queries: a base per-variant projection, plus a +separate protein-HGVS query stitched back in Python (see :func:`get_protein_hgvs_by_record` for why +it's split out). + +Each record carries the **submitted** HGVS (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the depositor's +target frame) and the **mapped** (reference-frame) HGVS as a ``MappedTriple`` — one canonical HGVS per +level (``genomic``/``cdna``/``protein``), with ``assay_level`` naming the measured slot. A nucleotide +assay fills all three slots; a protein assay fills only ``protein`` (the c/g fan-out is ambiguous, so +no pick is fabricated). This is the canonical projection only, never the reverse-translation fan-out. + +The nucleotide pair comes from the authoritative (measured) allele plus its ``projection_group`` +partner at the other nucleotide level (a group has ≤2 members, so this stays one row per variant). +Where ``projection_group`` is unpopulated (pre-reverse-translation data), that slot is ``None`` until +reprocessed. Does not assemble Cat-VRS. + +``as_of`` reconstructs the annotation layer at a past instant; submitted HGVS and scores are immutable +and unaffected. Defaults to currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.hgvs import parse_simple_substitution +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class HgvsField: + """An HGVS string plus its parsed position/ref/alt when it's a placeable simple substitution. + + ``position``/``ref``/``alt`` are ``None`` for splice/intronic, indels, frameshifts, and + multivariants — those carry ``hgvs`` alone. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + +@dataclass(frozen=True) +class MappedTriple: + """The canonical mapped (reference-frame) HGVS at each level, or ``None`` where absent. + + A nucleotide assay populates all three slots (measured + the authoritative allele's + ``projection_group`` partner + protein apex); a protein assay populates only ``protein`` — the + c/g fan-out is ambiguous, so no pick is fabricated for ``cdna``/``genomic``. + + ``cdna`` is the level-invariant field: populated whether the assay was measured at ``cdna`` or + ``genomic``, so it's the field to key cross-assay searches on. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + +@dataclass(frozen=True) +class LeanVariantRecord: + """One pre-chewed per-variant record, keyed by ``variant_urn``. + + ``consequence`` is a representative (most severe) VEP call. ``clingen_allele_id``/``assay_level_digest`` + bridge into the annotation dimensions. ``mapped`` is the canonical :class:`MappedTriple`; + ``assay_level`` names which of its slots is the measured one. Any field is ``None`` when its + source is absent (an unmapped variant → null ``assay_level`` and an empty triple). + """ + + variant_urn: str + score: Optional[float] + consequence: Optional[str] + clingen_allele_id: Optional[str] + assay_level_digest: Optional[str] + hgvs_nt: Optional[HgvsField] + hgvs_pro: Optional[HgvsField] + hgvs_splice: Optional[HgvsField] + assay_level: Optional[SequenceLevel] + mapped: MappedTriple + + +def _level(value: Optional[str]) -> Optional[SequenceLevel]: + """Coerce a stored assay-level string into :class:`SequenceLevel`, or ``None`` if absent.""" + return SequenceLevel(value) if value else None + + +def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: + """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable + simple substitution. ``None`` for an absent string.""" + if not hgvs: + return None + + block = parse_simple_substitution(hgvs) + if block is None: + return HgvsField(hgvs=hgvs) + + return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) + + +def mapped_hgvs_by_level( + *, + assay_level: Optional[SequenceLevel], + assay_level_hgvs: Optional[str], + projection_level: Optional[str], + projection_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> dict[str, Optional[str]]: + """The canonical mapped HGVS string at each populated level, keyed by ``SequenceLevel`` value. + + - Measured slot ← ``assay_level_hgvs``. + - Other nucleotide slot ← the authoritative link's ``projection_group`` partner (nucleotide assays + only, and only where the pairing is recorded). + - ``protein`` slot ← the separate protein-apex subquery. + + A protein assay fills only ``protein``. An unmapped variant (``assay_level is None``) yields an + empty mapping. + """ + slots: dict[str, Optional[str]] = {} + if assay_level == SequenceLevel.protein: + slots[SequenceLevel.protein.value] = assay_level_hgvs + elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): + slots[assay_level.value] = assay_level_hgvs + slots[SequenceLevel.protein.value] = protein_hgvs + # The other nucleotide level, from the projection_group partner (null where unpopulated). + if projection_level is not None: + slots[projection_level] = projection_hgvs + + return slots + + +def _mapped_triple( + *, + assay_level: Optional[SequenceLevel], + assay_level_hgvs: Optional[str], + projection_level: Optional[str], + projection_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> MappedTriple: + """Assemble the canonical :class:`MappedTriple` for one variant, wrapping each slot from + :func:`mapped_hgvs_by_level` as an :class:`HgvsField`.""" + slots = mapped_hgvs_by_level( + assay_level=assay_level, + assay_level_hgvs=assay_level_hgvs, + projection_level=projection_level, + projection_hgvs=projection_hgvs, + protein_hgvs=protein_hgvs, + ) + return MappedTriple( + genomic=_hgvs_field_for_str(slots.get(SequenceLevel.genomic.value)), + cdna=_hgvs_field_for_str(slots.get(SequenceLevel.cdna.value)), + protein=_hgvs_field_for_str(slots.get(SequenceLevel.protein.value)), + ) + + +def get_protein_hgvs_by_record( + db: Session, + score_set_id: Optional[int] = None, + *, + variant_ids: Optional[Sequence[int]] = None, + as_of: Optional[datetime] = None, +) -> dict[int, str]: + """The canonical protein-level mapped HGVS for a set of live mapping records, keyed by + ``mapping_record.id``. Records with no protein projection (UTR/intronic) are absent from the map. + Shared by the lean whole-set view and the CSV export. + + Exactly one of *score_set_id* (every variant in one score set) or *variant_ids* (an explicit set, + which may span several score sets — the variant-level CSV's cross-set equivalent measurements) is + required. + + Runs as its own query rather than a join: the ``DISTINCT ON`` cardinality is opaque to the planner, + so a join lands it on the inner side of a nested loop and rescans it once per variant — O(N^2), and + historically ~97% of the lean endpoint's runtime. Stitching by ``mapping_record_id`` in Python keeps + both queries O(N). + """ + if (score_set_id is None) == (variant_ids is None): + raise ValueError("exactly one of score_set_id or variant_ids must be provided") + + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_statement = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id/variant_id, to scope the query. + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set (or this explicit variant set) so the cost scales with the response. + .where( + prot_variant.score_set_id == score_set_id + if score_set_id is not None + else prot_variant.id.in_(variant_ids or []) + ) + # Live links only; a live link implies a live parent record (ValidTime invariant), so no + # separate prot_record.live_at check is needed. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. + .where(prot_allele.level == SequenceLevel.protein.value) + # DISTINCT ON needs ORDER BY to lead with its column; allele.id breaks ties (inert today — + # only one protein allele exists per record). + .order_by(prot_link.mapping_record_id, prot_allele.id) + ) + # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). + return {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + + +def get_lean_score_set_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> list[LeanVariantRecord]: + """Assemble the lean whole-set view for ``score_set``, one record per variant, ordered by variant + number. Unmapped variants are retained (left joins, null mapped fields) so the table and score + histogram still see them. ``as_of`` defaults to currently-live rows. + """ + # The authoritative link's *other* projection_group member — same record, same group, not + # authoritative — carries the canonical HGVS at the unmeasured nucleotide level. See join below. + projection_link = aliased(MappingRecordAllele) + projection_allele = aliased(Allele) + + # 1:1 annotation joins, so this is a stream of O(N) index-scan lookups. MappingRecord.id rides + # along as the key the protein projection (separate query, below) is stitched back on. + base_statement = ( + select( + Variant.urn, + Variant.data, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, + MappingRecord.hgvs_assay_level, + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + projection_allele.level.label("projection_level"), + # Exactly one of hgvs_g/hgvs_c is populated (it's a nucleotide allele); coalesce picks it. + func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), + ) + # LEFT join so unmapped variants stay (null mapped fields) for the table + score histogram. + # live_at rides in the ON clause, not WHERE. a WHERE would reject the outer join's null row, + # silently turning this into an INNER join. Same pattern on every join below. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + # The one authoritative (measured) allele; is_authoritative keeps this 1:1 (one authoritative + # link per live record, an invariant the mapping job upholds). + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + # The allele row itself (digest, ClinGen id). No live_at: alleles are immutable/deduplicated, + # not ValidTime — the live link already establishes what applies. + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + # Its VEP consequence. + .outerjoin( + VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + ) + # The authoritative link's projection_group partner (other nucleotide level). Unlike the protein + # subquery, this is safe as a direct join: a group has ≤2 members, so at most one match, and the + # mapping_record_id predicate uses an existing index — no O(N^2) rescan. A NULL group (protein + # assay, or pre-reverse-translation data) matches nothing, so the slot degrades to None. + .outerjoin( + projection_link, + and_( + projection_link.mapping_record_id == MappingRecord.id, + projection_link.projection_group == MappingRecordAllele.projection_group, + projection_link.is_authoritative.is_(False), + projection_link.live_at(as_of), + ), + ) + .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) + # The anchor: just this score set's variants. + .where(Variant.score_set_id == score_set.id) + # Natural table order: variant number (the integer after '#'), id breaks ties stably. + .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) + ) + + # Runs as its own query, not a join (see get_protein_hgvs_by_record), stitched back below. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) + + return [ + LeanVariantRecord( + variant_urn=row.urn, + score=score_from_variant_data(row.data), + consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + assay_level_digest=row.vrs_digest, + hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), + hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), + hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), + assay_level=_level(row.assay_level), + mapped=_mapped_triple( + assay_level=_level(row.assay_level), + assay_level_hgvs=row.hgvs_assay_level, + projection_level=row.projection_level, + projection_hgvs=row.projection_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ), + ) + for row in db.execute(base_statement) + ] diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 698bc515a..002832c61 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,5 +1,6 @@ import logging from collections import Counter, defaultdict +from datetime import datetime from operator import attrgetter from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence @@ -23,6 +24,7 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list +from mavedb.models.allele import Allele from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -32,7 +34,8 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier from mavedb.models.refseq_offset import RefseqOffset @@ -541,35 +544,50 @@ def find_publish_or_private_superseded_score_set_tail( return score_set -def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) -> Sequence[MappedVariant]: - """ - Load the current mapped variants for a score set with the relationships required to build VA-Spec - annotations eagerly loaded. - - This is the single source of truth for the eager-load shape shared by the annotated-variant - streaming endpoints and the public data export. The annotation builders reach through - ``MappedVariant.variant.score_set`` for publications, contributors, license, experiment, and score - calibrations, so each of those is loaded up front to avoid per-variant lazy loads. +def get_annotatable_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> Sequence[Variant]: + """Load a score set's annotatable variants from its variant's allele-graph, with the score-set + relationships the VA-Spec builders need eagerly loaded. + + A variant is *annotatable* when it has a live ``MappingRecord`` whose authoritative allele carries a + ``post_mapped`` VRS — exactly the condition under which :func:`annotation.context.variant_annotation_context` + yields a context. This is the single source of truth for the eager-load shape shared by the + annotated-variant streaming endpoints and the public data export. The annotation builders reach through + ``variant.score_set`` for publications, contributors, license, experiment, and score calibrations, so + each of those is loaded up front to avoid per-variant lazy loads. + + ``as_of`` reconstructs the annotatable set as it stood at a past instant: the same half-open + ``[valid_from, valid_to)`` predicate is applied to both the record and the authoritative link, so the + set is evaluated at one moment (matching :func:`annotation.context.variant_annotation_context`'s own + ``as_of``, so a per-variant context can be rebuilt at the same instant downstream). Defaults to the + currently-live rows. """ return ( - db.query(MappedVariant) - .join(MappedVariant.variant) - .join(Variant.score_set) - .filter(Variant.score_set_id == score_set.id) - .filter(MappedVariant.current.is_(True)) - .options( - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.publication_identifier_associations), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.created_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.modified_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.license), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.experiment), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.score_calibrations), + db.scalars( + select(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, and_(Allele.id == MappingRecordAllele.allele_id, Allele.post_mapped.isnot(None))) + .where(Variant.score_set_id == score_set.id) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.publication_identifier_associations), + selectinload(Variant.score_set).selectinload(ScoreSet.created_by), + selectinload(Variant.score_set).selectinload(ScoreSet.modified_by), + selectinload(Variant.score_set).selectinload(ScoreSet.license), + selectinload(Variant.score_set).selectinload(ScoreSet.experiment), + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + ) + .distinct() ) + .unique() .all() ) diff --git a/src/mavedb/lib/term_systems.py b/src/mavedb/lib/term_systems.py new file mode 100644 index 000000000..b9998433a --- /dev/null +++ b/src/mavedb/lib/term_systems.py @@ -0,0 +1,36 @@ +"""Shared building block for GA4GH ``Coding`` objects drawn from a named term system. + +A :data:`TermSystem` pairs a system URI/CURIE with the short prefix used to build a ``Coding.id`` for +codes drawn from it, kept together so a term's id and system can't drift apart the way two independent +lookup tables could. +""" + +from ga4gh.core.models import Coding + +TermSystem = tuple[str, str] + + +### MaveDB Term Systems + +# MaveDB's own Cat-VRS relation codes (member -> defining), in cat_vrs.py's own framing rather than the +# spec's. +MAVEDB_CAT_VRS_RELATION: TermSystem = ("https://mavedb.org/cat-vrs/relations", "mavedb") + + +### External Term Systems + +# Sequence Ontology. Used by several Cat-VRS Relation terms (translation_of, transcribed_to); see +# ga4gh/cat-vrs examples/json/proteinSequenceConsequence-ex2.json. +SEQUENCE_ONTOLOGY: TermSystem = ("http://www.sequenceontology.org", "so") + +# Cat-VRS's own internally controlled vocabulary for the allele-relation terms that have no external +# ontology equivalent (currently just liftover_to). See ga4gh/cat-vrs recipes-source.yaml. Other +# ga4gh-gks-term categories (e.g. experimental-var-func-impact-classification) are separate term +# systems under the same vocabulary owner, not this one. +GKS_ALLELE_RELATION: TermSystem = ("ga4gh-gks-term:allele-relation", "ga4gh-gks-term") + + +def coding(term_system: TermSystem, code: str) -> Coding: + """Build a ``Coding`` for `code`, drawn from `term_system`.""" + uri, prefix = term_system + return Coding(id=f"{prefix}:{code}", code=code, system=uri) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py new file mode 100644 index 000000000..1ad8fdf14 --- /dev/null +++ b/src/mavedb/lib/variant_detail.py @@ -0,0 +1,268 @@ +"""Assembles the variant-detail envelope backing ``GET /variants/{urn}``. + +The envelope has two tiers: flat, UI-ergonomic assay-level fields (HGVS pair, digest, ClinGen id), +and a spec-pure GA4GH ``CategoricalVariant`` (built by :mod:`lib.cat_vrs`) with a MaveDB layer +riding alongside it, keyed by VRS digest — per-allele identity (level, HGVS, ClinGen id, +member→defining relation) and external annotations (:mod:`lib.allele_annotations`). Also includes +the variant's functional ``classifications`` per calibration, and its version standing +(``is_current`` / ``superseded_by_score_set``), so a superseded variant self-describes. + +``as_of`` only reconstructs the **molecular** layer (Cat-VRS membership and VEP/gnomAD/ClinVar +annotations) at the past instant. Scores are immutable and calibrations carry no ``ValidTime``, so +both are always returned as they stand now, regardless of ``as_of``. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_encoding +from mavedb.lib.score_calibrations import calibration_preference_key +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class VariantClassificationRecord: + """One functional classification the variant belongs to, tagged with its calibration. + + ``calibration_id`` / ``primary`` locate it among the score set's calibrations (``primary`` is + the UI default). ``classification`` is the ORM + :class:`ScoreCalibrationFunctionalClassification`, serialized at the view-model boundary. + """ + + calibration_id: int + primary: bool + classification: ScoreCalibrationFunctionalClassification + + +@dataclass(frozen=True) +class VariantDetail: + """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" + + urn: str + scores: Optional[dict[str, Any]] + counts: Optional[dict[str, Any]] + classifications: list[VariantClassificationRecord] + + # Flat, UI-ergonomic assay-level fields. + assay_level: Optional[SequenceLevel] + target_hgvs: Optional[str] # submitted, target/assay coordinates + reference_hgvs: Optional[str] # mapped, reference coordinates, assay level + assay_level_digest: Optional[str] + clingen_allele_id: Optional[str] + + # Raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer doesn't need to dig the measured + # allele out of the Cat-VRS below. + pre_mapped: Optional[dict[str, Any]] + post_mapped: Optional[dict[str, Any]] + + # Spec-pure GA4GH Cat-VRS (no MaveDB fields), plus the MaveDB layer alongside it: a per-allele + # identity sidecar keyed by VRS digest, one entry per linked allele (shares keys with + # `annotations`), carrying level, HGVS, ClinGen id, and relation. + molecular_representation: Optional[dict[str, Any]] + mode: Optional[str] # CatVrsMode value — projection | reverse_translation + alleles: dict[str, AlleleIdentity] + + # External annotations, keyed by VRS digest, joined to the Cat-VRS members / the alleles sidecar. + annotations: dict[str, AlleleAnnotations] + + # Version standing — self-descriptive for a superseded variant. + is_current: bool + # Score-set URN (not variant URN) of the version that supersedes this one, if any and readable. + # Supersession is versioned at the score-set level, and a newer version may add/drop/renumber + # variants, so consumers must look this variant up within that score set rather than follow a + # superseding-variant pointer. + superseded_by_score_set: Optional[str] + + +def _classifications_for_variant( + db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] +) -> list[VariantClassificationRecord]: + """Functional classifications the variant belongs to, one row per calibration that classifies it. + + Membership is read from the calibration→classification→variant association table, ordered by + ``calibration_preference_key`` (+ id) so the first entry matches the UI's default. + ``visible_calibration_ids`` restricts to readable calibrations (``None`` = no restriction, for + lib-level callers). + """ + statement = ( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + ) + if visible_calibration_ids is not None: + statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) + + rows = sorted( + db.execute(statement).tuples().all(), key=lambda row: (*calibration_preference_key(row[1]), row[1].id) + ) + return [ + VariantClassificationRecord( + calibration_id=calibration.id, primary=calibration.primary, classification=classification + ) + for classification, calibration in rows + ] + + +def _derivation_for(*, assay_level: Optional[SequenceLevel], is_convergent: bool) -> AlleleDerivation: + """Provenance of a *non-focus* linked allele, derived from the assay level. + + The measured allele is the focus (no derivation). For every other allele: + + - **Protein assay** → ``CANDIDATE``: reverse translation is ambiguous, so the derived nucleotide + fan-out is only a candidate (which codon was actually measured is unknown). + - **Nucleotide assay, convergent encoding** (``is_convergent``) → ``CONVERGENT``: an nt member in + a different projection group. A distinct, precisely-known variant that happens to converge on + the same protein consequence, not a projection of the measured change. + - **Nucleotide assay, otherwise** → ``PROJECTION``: the deterministic class derived from the + measured change itself. + + ``is_convergent`` comes from :func:`cat_vrs.is_convergent_encoding`, keeping this ``derivation`` + axis aligned with the Cat-VRS ``relation`` axis (``co_encodes`` ↔ ``convergent``, + ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, ``encodes`` ↔ + ``candidate``). + """ + if assay_level == SequenceLevel.protein.value: + return AlleleDerivation.CANDIDATE + if is_convergent: + return AlleleDerivation.CONVERGENT + return AlleleDerivation.PROJECTION + + +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: + """Depositor-submitted HGVS in the variant's assay frame: ``hgvs_pro`` for a protein assay, + otherwise ``hgvs_nt`` (genomic and coding assays share this column; ``hgvs_splice`` is never an + index column).""" + if assay_level == SequenceLevel.protein.value: + return variant.hgvs_pro + return variant.hgvs_nt + + +def get_variant_detail( + db: Session, + variant: Variant, + *, + superseding_score_set: Optional[ScoreSet] = None, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetail: + """Assemble the variant-detail envelope for ``variant``. + + ``superseding_score_set`` is the newer version the caller already resolved for visibility + (``None`` if unreadable, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current`` / ``superseded_by_score_set``. ``visible_calibration_ids`` restricts + classifications to readable calibrations. ``as_of`` reconstructs the molecular layer only — + see the module docstring. + """ + data = variant.data if isinstance(variant.data, dict) else {} + scores = data.get("score_data") + counts = data.get("count_data") + + # The live mapping record supplies the assay level and the mapped (reference-frame) assay HGVS. + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + assay_level = SequenceLevel(record.assay_level) if record is not None else None + reference_hgvs = record.hgvs_assay_level if record is not None else None + + # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all + # linked alleles seed the digest-keyed annotation map. + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + authoritative = next((link for link in links if link.is_authoritative), None) + assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None + clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None + + annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) + + # Spec-pure Cat-VRS built on the fly, plus mode + per-member relations. The defining allele is + # deliberately absent from `relations`, so it gets relation=None below. + transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) + if transit is not None: + molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) + mode: Optional[str] = transit.mode.value + relations = transit.member_relations + else: + molecular_representation = None + mode = None + relations = {} + + # Per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same link + # set that seeds `annotations`, so the two maps share keys). Carries level, reference-frame HGVS + # (exactly one of hgvs_g/c/p is populated per allele), ClinGen id, and the member→defining relation. + + # projection_group -> the digests sharing it, for within-record projection grouping. A link's + # projection is the ≤1 *other* digest in its group; the protein apex and any pre-RT link carry a + # NULL group, so they appear in no bucket and projection_of stays None for them. + group_digests: dict[int, list[str]] = {} + for link in links: + if link.projection_group is not None and link.allele.vrs_digest is not None: + group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + + # Anchor for the convergence test: an nt member in a *different* projection group than the + # measured change is a convergent encoding (co_encodes / convergent), not a projection of it. + defining_group = authoritative.projection_group if authoritative is not None else None + + alleles: dict[str, AlleleIdentity] = {} + for link in links: + allele = link.allele + + # The other digest in this link's projection_group, if any (groups have ≤2 members). + projection_of: Optional[str] = None + if link.projection_group is not None: + projection_of = next( + (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), + None, + ) + + relation = relations.get(allele.vrs_digest) + is_convergent = is_convergent_encoding(allele.level, link.projection_group, defining_group=defining_group) + alleles[allele.vrs_digest] = AlleleIdentity( + level=allele.level, + hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, + clingen_allele_id=allele.clingen_allele_id, + is_focus=link.is_authoritative, + relation=relation.value if relation is not None else None, + derivation=( + None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_convergent=is_convergent) + ), + projection_of=projection_of, + ) + + return VariantDetail( + # TODO(#372) + urn=variant.urn or "", + scores=scores, + counts=counts, + classifications=_classifications_for_variant(db, variant, visible_calibration_ids=visible_calibration_ids), + assay_level=assay_level, + target_hgvs=_submitted_assay_level_hgvs(variant, assay_level), + reference_hgvs=reference_hgvs, + assay_level_digest=assay_level_digest, + clingen_allele_id=clingen_allele_id, + pre_mapped=record.pre_mapped if record is not None else None, + post_mapped=authoritative.allele.post_mapped if authoritative is not None else None, + molecular_representation=molecular_representation, + mode=mode, + alleles=alleles, + annotations=annotations, + is_current=superseding_score_set is None, + superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, + ) diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index c24c5ed95..31d89e6df 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -3,14 +3,22 @@ This module provides database operations for the variant_translations table, which stores relationships between protein allele (PA) and nucleotide allele (CA) ClinGen IDs. + +FROZEN (serving-only). The populate_variant_translations_for_score_set job that wrote this table was +retired in the #742 migration: the reverse-translation allele equivalence space (genomic/coding/protein +VRS alleles per variant, linked via MappingRecordAllele with HGVS on Allele) now covers PA<->CA +relationships without querying ClinGen. These helpers and the variant_translations table remain only to +serve existing old-model data; they are never written for new score sets and are dropped at read-cutover. """ from typing import cast +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session +from mavedb.models.allele import Allele from mavedb.models.variant_translation import VariantTranslation @@ -39,3 +47,21 @@ def upsert_variant_translations(db: Session, translations: list[tuple[str, str]] created = result.rowcount existing = len(unique) - created return created, existing + + +def get_or_create_allele(db: Session, allele_draft: Allele) -> Allele: + """Return the existing Allele matching ``allele_draft``'s vrs_digest, else add the draft. + + This is a get-or-create, not an upsert: a matching row is returned untouched, and on + a miss the draft is added to the session (not flushed). The draft is never used to + update an existing row. + + NOTE: This function does not persist the returned Allele to the database; the caller + is responsible for committing the session. + """ + existing = db.scalars(select(Allele).where(Allele.vrs_digest == allele_draft.vrs_digest)).one_or_none() + if existing is not None: + return existing + + db.add(allele_draft) + return allele_draft diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index 487d101aa..ba90886b8 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -1,54 +1,98 @@ import re from typing import Any, Optional +from mavedb.lib.hgvs import join_cis_phased_hgvs +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN, VARIANT_SCORE_DATA from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant + +def score_from_variant_data(data: Optional[Any]) -> Optional[float]: + """The canonical numeric score in a variant's ``data`` JSONB (``score_data.score``), or ``None``. + + The score column is required for every score set, but an individual variant may carry a null/NA + score. Returns ``None`` when the score is absent, null, or not coercible to a float; a numeric + string (``"1.5"``) coerces, but ``bool`` is rejected — a JSON ``true`` is not a score. Robust to + malformed JSONB: a non-mapping ``data`` or ``score_data`` yields ``None`` rather than raising. + Operates on the ``data`` mapping directly so it serves both ORM objects and bare ``Variant.data`` + column reads. + """ + if not isinstance(data, dict): + return None + + score_data = data.get(VARIANT_SCORE_DATA) + if not isinstance(score_data, dict): + return None + + value = score_data.get(REQUIRED_SCORE_COLUMN) + if value is None or isinstance(value, bool): + return None + + try: + return float(value) + except (TypeError, ValueError): + return None + + +def variant_score(variant: Variant) -> Optional[float]: + """The canonical numeric score of a variant (see :func:`score_from_variant_data`).""" + return score_from_variant_data(variant.data) + + HGVS_G_REGEX = re.compile(r"(^|:)g\.") HGVS_P_REGEX = re.compile(r"(^|:)p\.") -def hgvs_from_vrs_allele(allele: dict) -> str: +def hgvs_from_vrs_allele(allele: dict) -> Optional[str]: """ - Extract the HGVS notation from the VRS allele. + Extract the HGVS notation from the VRS allele, or None if it carries no expression. """ try: - # VRS 2.X - return allele["expressions"][0]["value"] + expressions = allele["expressions"] # VRS 2.X except KeyError: if "variation" in allele: raise ValueError("VRS 1.X format not supported.") # VRS 1.X. We don't want to allow this. - # return allele["variation"]["expressions"][0]["value"] - else: - raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + + # A valid VRS allele may simply carry no HGVS expression (None or empty) — e.g. a member of a + # cis-phased block. That is "no HGVS", not a crash. + if not expressions: + return None + return expressions[0]["value"] + +def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: + """Extract a single HGVS string from a post-mapped VRS object. -def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: + Multi-variant blocks (Haplotype/CisPhasedBlock) are cis-phased, so their members combine + into one bracketed expression (``NC_…:g.[a;b]``) when ``combine_cis`` is set. It defaults + off because some consumers cannot yet handle a bracketed expression — notably ClinGen + submission, which has no single CAID for a multi-variant cis block (see + https://github.com/VariantEffect/mavedb-api/issues/764). + """ if not post_mapped_vrs: return None - if post_mapped_vrs["type"] == "Haplotype": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "CisPhasedBlock": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "Allele": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(post_mapped_vrs)] + if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore + members = post_mapped_vrs["members"] + elif post_mapped_vrs["type"] in ("Allele", "VariationDescriptor"): # type: ignore + members = [post_mapped_vrs] else: return None - if len(variations_hgvs) == 0: - return None - # raise ValueError(f"No variations found in variant {variant_urn}.") + member_hgvs = [hgvs_from_vrs_allele(allele) for allele in members] - # TODO (https://github.com/VariantEffect/mavedb-api/issues/468) In a future version, we will be able to generate - # a combined HGVS string for haplotypes and cis phased blocks directly from mapper output. - if len(variations_hgvs) > 1: + # No members, or a member carrying no HGVS expression — no single/combinable HGVS to return. + if not member_hgvs or any(h is None for h in member_hgvs): return None - # raise ValueError(f"Multiple variations found in variant {variant_urn}.") - return variations_hgvs[0] + hgvs_values: list[str] = [h for h in member_hgvs if h is not None] + if len(hgvs_values) > 1: + return join_cis_phased_hgvs(hgvs_values) if combine_cis else None + + return hgvs_values[0] def get_id_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index a7d4e7b3c..25c0732fa 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -4,17 +4,22 @@ import functools import logging import os -from typing import Optional, Sequence +from datetime import date +from enum import Enum +from typing import Mapping, NamedTuple, Optional, Sequence -import requests +from sqlalchemy import select +from sqlalchemy.orm import Session +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import request_with_backoff +from mavedb.models.vep_allele_consequence import VepAlleleConsequence logger = logging.getLogger(__name__) + ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org") -# List of all possible VEP consequences, in order from most to least severe VEP_CONSEQUENCES = [ "transcript_ablation", "splice_acceptor_variant", @@ -72,6 +77,40 @@ "intron_variant", "intergenic_variant", ] +""" +List of all functional consequences VEP can return, in order of severity (most severe first). +""" + + +class VepLinkVerdict(str, Enum): + """Per-allele outcome of a VEP linking run, returned for every allele whose status is decided. + + The single source of truth for what happened to an allele's consequence this run — the caller + derives annotation status from this, never by re-querying consequence state. An allele absent from + the map had no live consequence and resolved none this run (the caller reads that as "no result"). + + - ``CREATED`` — a new or changed consequence was created/superseded this run. + - ``UNCHANGED`` — a live consequence was retained (value matched, or held against a null run). + """ + + CREATED = "created" + UNCHANGED = "unchanged" + + +class VepResolution(NamedTuple): + """Outcome of resolving a set of HGVS strings, splitting the two kinds of "no consequence". + + This outcome allows us to differentiate between a genuine empty (VEP found nothing) and an + unknown (VEP failed to answer). + + - ``consequences`` — HGVS that resolved to a most-severe consequence (the hits). + - ``errored`` — HGVS whose VEP/Recoder request *failed* (HTTP/transport error after retries); the + result is unknown and the allele should be retried, not treated as a negative. + - Any queried HGVS in neither set was answered (HTTP 200) with no consequence — a genuine **empty**. + """ + + consequences: dict[str, str] + errored: set[str] async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str]]: @@ -81,35 +120,30 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str missing_hgvs (Sequence[str]): List of HGVS strings to recode. Returns: - dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). - Returns an empty dict if Ensembl rejects the batch (e.g. 400 for - unrecognised identifiers) — callers treat missing entries as failures. + dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). An input + with no recodable genomic mapping is simply absent (a genuine empty). + + Raises: + requests.exceptions.RequestException: if the Recoder request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ headers = {"Content-Type": "application/json", "Accept": "application/json"} + # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/variant_recoder/human", - headers=headers, - json={"ids": list(missing_hgvs)}, - timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier format) means the batch - # cannot be recoded. Return empty so callers can handle these missing entries. - logger.warning( - f"Variant Recoder returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(missing_hgvs)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return {} + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/variant_recoder/human", + headers=headers, + json={"ids": list(missing_hgvs)}, + timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed + ), + ) data = response.json() # request_with_backoff handles http errors, so no need to check response status @@ -119,11 +153,13 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str hgvs_string = variant_data.get("input") if isinstance(variant_data, dict) else None if variant_str == "input" or not hgvs_string: continue + genomic_strings = variant_data.get("hgvsg") if isinstance(variant_data, dict) else None if genomic_strings: for genomic_hgvs in genomic_strings: if genomic_hgvs.startswith("NC_"): hgvs_to_genomic.setdefault(hgvs_string, []).append(genomic_hgvs) + return hgvs_to_genomic @@ -138,11 +174,14 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O hgvs_strings (Sequence[str]): List of HGVS strings to process (max 200 per call). Returns: - dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. - If no consequence found, maps to None. Returns an empty dict - if Ensembl rejects the batch (e.g. 400 for unrecognised - identifiers) — callers treat missing entries as needing Recoder - fallback or as failures. + dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. An HGVS the + successful response carried no consequence for maps to None (a genuine + miss — the caller may try Recoder, else treats it as empty). + + Raises: + requests.exceptions.RequestException: if the VEP request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ if len(hgvs_strings) > 200: raise ValueError( @@ -155,27 +194,17 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/vep/human/hgvs", - headers=headers, - json={"hgvs_notations": list(hgvs_strings)}, - timeout=60, # VEP can be slow for large batches. - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier) means the batch cannot - # be resolved. Return empty so the callers can handle these missing entries. - logger.warning( - f"VEP returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(hgvs_strings)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return result + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/vep/human/hgvs", + headers=headers, + json={"hgvs_notations": list(hgvs_strings)}, + timeout=60, # VEP can be slow for large batches. + ), + ) data = response.json() for entry in data: @@ -185,3 +214,113 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O result[hgvs] = most_severe_consequence return result + + +async def get_ensembl_release() -> str: + """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). + + An Ensembl release is coordinated — software, transcript set, and consequence vocabulary all bump + together under one number — so this single value version-keys VEP results the way gnomAD keys on its + data version. The job stamps it on each consequence and skips re-querying alleles already live at the + current release. Raises on failure: the version is load-bearing for the skip, so a job that cannot + determine it must abort rather than mis-version its writes. + """ + headers = {"Content-Type": "application/json", "Accept": "application/json"} + loop = asyncio.get_running_loop() + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="GET", + url=f"{ENSEMBL_API_URL}/info/software", + headers=headers, + timeout=30, + ), + ) + return str(response.json()["release"]) + + +def link_vep_consequences_to_alleles( + db: Session, + consequence_by_allele_id: Mapping[int, Optional[str]], + *, + source_version: str, + access_date: date, +) -> dict[int, VepLinkVerdict]: + """Store VEP consequences against deduplicated alleles, superseding only on change. + + ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run + (``None`` when VEP + Variant Recoder found nothing). ``source_version`` is the Ensembl release the + run resolved against. Each allele holds at most one live :class:`VepAlleleConsequence`, handled per + allele: + + - **unchanged** (live row already carries this consequence): advance ``source_version`` and + ``access_date`` in place — no supersede. Supersede is value-keyed, not version-keyed: a new + release that resolves the same categorical consequence must not fabricate a transaction-time + boundary, which would churn history every release. + - **new or changed** (no live row, or a different consequence): supersede keyed on ``allele_id`` + (retire the old, insert the successor stamped with this ``source_version``/``access_date``). + - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. + Log a warning if VEP found no consequence for an allele which previously had a live consequence. + + Does not commit. Returns a verdict per allele whose status is decided this run: + :attr:`VepLinkVerdict.CREATED` for a created/superseded consequence, :attr:`~VepLinkVerdict.UNCHANGED` + for a live consequence retained (value matched, or held against a null run). An allele absent from + the map had no live row and resolved nothing — the caller reads that as "no result". This is the + single source of truth for per-allele status; callers must not re-derive it from consequence state. + """ + save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) + logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) + + verdicts: dict[int, VepLinkVerdict] = {} + for allele_id, consequence in consequence_by_allele_id.items(): + live = db.scalar( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ) + + # TODO#780 - VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained + # consequence is UNCHANGED (status preexisting), while no live row at all leaves the allele out of + # the map (the caller reads that as a no-result). + if consequence is None: + if live is not None: + logger.warning( + f"VEP found no consequence for allele {allele_id} this run; leaving prior consequence " + f"'{live.functional_consequence}' in place.", + extra=logging_context(), + ) + verdicts[allele_id] = VepLinkVerdict.UNCHANGED + + continue + + # Unchanged: advance version/freshness in place. + if live is not None and live.functional_consequence == consequence: + live.source_version = source_version + live.access_date = access_date + verdicts[allele_id] = VepLinkVerdict.UNCHANGED + continue + + # New or changed consequence: retire any live row, insert the successor. + VepAlleleConsequence.supersede_live_where( + db, + [ + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + ) + ], + VepAlleleConsequence.allele_id == allele_id, + ) + verdicts[allele_id] = VepLinkVerdict.CREATED + + changed_allele_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) + logger.info( + msg=f"Created or superseded {changed_allele_count} VEP allele consequences this run.", + extra=logging_context(), + ) + return verdicts diff --git a/src/mavedb/lib/vrs.py b/src/mavedb/lib/vrs.py new file mode 100644 index 000000000..c5ee00ee1 --- /dev/null +++ b/src/mavedb/lib/vrs.py @@ -0,0 +1,134 @@ +"""Deserialization of stored ``post_mapped`` JSONB into GA4GH VRS objects. + +The mapping pipeline writes each allele's mapped representation as a VRS-shaped dict in +``Allele.post_mapped``. These helpers rehydrate that dict into strict ``ga4gh.vrs`` Pydantic models for the +Cat-VRS transit, the variant-detail envelope, and the VA-Spec annotation subjects. +""" + +from ga4gh.core.models import Extension +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LengthExpression, + LiteralSequenceExpression, + MolecularVariation, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + + +def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: + """ + Converts a dictionary containing allelic mapping results into an Allele object. + + This function handles the possibility of an extra nesting level in early VRS 1.3 objects, + where Allele objects are contained within a `variation` property. If the `variation` key + is not present, the function assumes the dictionary itself represents the variation. + + Args: + allelic_mapping_results (dict): A dictionary containing allelic mapping results. + It may include a `variation` key or directly represent the variation. + + Returns: + Allele: An Allele object constructed from the provided mapping results. + + Raises: + KeyError: If required keys are missing from the input dictionary. + """ + + # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects + # are contained in a `variation` property. Although it's unlikely variants of this form + # will ever be exported in this format, we handle the possibility. + try: + variation = allelic_mapping_results["variation"] + except KeyError: + variation = allelic_mapping_results + + state_dict = variation["state"] + state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression + if state_dict.get("type") == "ReferenceLengthExpression": + state = ReferenceLengthExpression(**state_dict) + elif state_dict.get("type") == "LengthExpression": + state = LengthExpression(**state_dict) + elif state_dict.get("type") == "LiteralSequenceExpression": + state = LiteralSequenceExpression(**state_dict) + else: + raise ValueError( + f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " + "Update allele_from_mapped_variant_dictionary_result to handle this type." + ) + + # Mapping results were not guaranteed to be generated on this version of VRS. + # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on + # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid. In such cases, + # using model_validate() directly is not possible. + return Allele( + id=variation.get("id"), + state=state, + digest=variation.get("digest"), + location=SequenceLocation( + start=variation.get("location", {}).get("start"), + end=variation.get("location", {}).get("end"), + digest=variation.get("location", {}).get("digest"), + id=variation.get("location", {}).get("id"), + sequenceReference=SequenceReference( + name=variation.get("location", {}).get("sequenceReference", {}).get("name"), + refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), + ), + ), + extensions=[ + Extension( + id=extension.get("id"), + name=extension["name"], + description=extension.get("description"), + value=extension.get("value"), + ) + for extension in variation.get("extensions", []) + ], + expressions=[ + Expression( + id=expression.get("id"), + syntax=expression["syntax"], + syntax_version=expression.get("syntax_version"), + value=expression["value"], + ) + for expression in variation.get("expressions", []) + ], + ) + + +def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: + """ + Extracts a VRS (Variation Representation Specification) object from a stored ``post_mapped`` dict. + + This function processes a dictionary of mapping results and returns a VRS object, + which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object + returned depends on the "type" field in the input dictionary. + + Args: + mapping_results (dict): A dictionary containing the mapping results of a variant. + It must include a "type" key indicating the type of VRS object + ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). + If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also + include a "members" key containing a list of member alleles. + + Returns: + MolecularVariation: A VRS object representing the mapped variant. This will be + either a `CisPhasedBlock` containing its member variants or an `Allele` + derived from the mapping results. + + Raises: + KeyError: If required keys are missing from the `mapping_results` dictionary. + """ + if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": + return MolecularVariation( + # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) + CisPhasedBlock( + id=None, + members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], + ) + ) + + return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py new file mode 100644 index 000000000..7979cdaec --- /dev/null +++ b/src/mavedb/lib/vrs_utils.py @@ -0,0 +1,221 @@ +"""VRS allele identification helpers. + +Centralizes the digest-correctness invariant for GA4GH VRS alleles: the +``ga4gh_identify`` Merkle tree caches sub-object digests on the object after +first identification, so any subsequent mutation (refgetAccession swap, +normalization, state coercion) leaves a stale id unless the cached digests +are cleared first. All allele identification in dcd_mapping must route +through :func:`identify_allele` so the digest is always recomputed from +current content. +""" + +from itertools import cycle +from typing import Any + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.extras.translator import AlleleTranslator +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LiteralSequenceExpression, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, + Syntax, +) +from ga4gh.vrs.normalize import normalize + +from mavedb.lib.hgvs import split_cis_phased_hgvs + +# HGVS type letter (``accession:g.``) → VRS Expression syntax. +_HGVS_SYNTAX_BY_TYPE = { + "g": Syntax.HGVS_G, + "c": Syntax.HGVS_C, + "p": Syntax.HGVS_P, + "n": Syntax.HGVS_N, + "m": Syntax.HGVS_M, + "r": Syntax.HGVS_R, +} + + +def _hgvs_syntax(hgvs: str) -> Syntax: + """Map an HGVS string to its VRS Expression syntax via the type letter after the accession.""" + _, _, rest = hgvs.partition(":") + try: + return _HGVS_SYNTAX_BY_TYPE[rest[:1]] + except KeyError: + raise ValueError(f"Cannot determine HGVS syntax for {hgvs!r}") + + +def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: + """Convert HGVS variation description to VRS object. + + The AlleleTranslator is supplied by the caller and reused across calls. ga4gh's + AlleleTranslator opens a UTA connection lazily on first translate (via HgvsTools) + and holds it for its lifetime, so constructing one per call opens — and leaks — a + UTA connection per variant, exhausting the server's slot budget. Build one per + job/worker and pass it in. + + :param hgvs: MAVE-HGVS variation string + :param translator: caller-owned AlleleTranslator backed by a sequence/refget proxy + :return: Corresponding VRS allele as a Pydantic class + """ + # coerce tmp HGVS string into formally correct term + if hgvs.startswith("NC_") and ":c." in hgvs: + hgvs = hgvs.replace(":c.", ":g.") + + allele: Allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) + + if ( + not isinstance(allele.location, SequenceLocation) + or not isinstance(allele.location.start, int) + or not isinstance(allele.location.end, int) + or not isinstance(allele.state, LiteralSequenceExpression) + ): + raise ValueError + + return allele + + +def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Allele | CisPhasedBlock: + """Translate an HGVS expression — possibly a cis-phased multivariant — into a VRS object. + + Mirrors dcd_mapping's ``vrs_map._construct_vrs_allele``: each component HGVS is translated + to an Allele independently; a single component returns a bare Allele, while two or more are + wrapped in a CisPhasedBlock. The reverse-translation job emits bracketed genomic forms + (``g.[a;b]``) for non-adjacent codon components that ga4gh's AlleleTranslator cannot + translate directly, so splitting and recombining is the only way to represent them. + + The block's GA4GH digest is order-independent, so the same biological cis-phased set always + identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component + ordering. + + Every component is normalized and re-identified through :func:`normalize_and_identify` + before use. ``translate_from`` is called with ``do_normalize=False`` and stamps ``id`` via + plain ``ga4gh_identify`` on the reused translator, so without this step a component carries a + non-canonical digest computed from the unnormalized object and from the Merkle tree's cached + sub-object digests — distinct biological variants can then collide onto one ``vrs_digest`` and + be merged by the digest-keyed ``get_or_create_allele``. Normalizing here also keeps RT digests + consistent with the mapper's, which is what lets the same allele dedup across sources. + + :param hgvs: a single- or cis-phased-multivariant HGVS string + :param translator: caller-owned AlleleTranslator reused across calls + :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set + """ + members = [] + for component in split_cis_phased_hgvs(hgvs): + allele = normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + # Stamp the source HGVS as the allele's expression so post_mapped is self-describing. + # Mirrors the mapper's authoritative alleles. + allele.expressions = [Expression(syntax=_hgvs_syntax(component), value=component)] + members.append(allele) + + if len(members) == 1: + return members[0] + + block = CisPhasedBlock(members=members) # type: ignore[call-arg] + block.id = identify_variation(block) + return block + + +def identify_allele(allele: Allele) -> str: + """Clear cached digests and return a fresh GA4GH identifier for *allele*. + + ``ga4gh_identify`` is a Merkle-tree: it calls ``get_or_create_digest`` on + sub-objects, returning any cached value without recomputing. Clearing both + the location digest and the allele digest first ensures the id is always + derived from the current object content — not from a value set before a + refgetAccession mutation or normalization. + + ``id`` is recomputed as well: ``ga4gh_identify`` returns a non-empty ``id`` as-is + (its default ``in_place`` only fills an *empty* id), so an allele that already + carries one — e.g. stamped by an ``identify=True`` ``AlleleTranslator`` — would + otherwise keep its stale id even with the digests cleared. Use in_place="always" + to force it to be recomputed from the content-derived digest. + """ + if isinstance(allele.location, SequenceLocation): + allele.location.digest = None + + allele.digest = None + digest = ga4gh_identify(allele, in_place="always") + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 + + return digest + + +def identify_variation(variation: Allele | CisPhasedBlock) -> str: + """Clear cached digests and return a fresh GA4GH id for an Allele or CisPhasedBlock. + + Generalizes :func:`identify_allele` to cis-phased blocks. A block's Merkle digest is + derived from its members' digests, so a stale member digest would silently propagate into + the block id. Clear every member (and its location) plus the block itself before + identifying so the id always reflects current content. + """ + if isinstance(variation, Allele): + return identify_allele(variation) + + for member in variation.members: + if isinstance(member, Allele): + if isinstance(member.location, SequenceLocation): + member.location.digest = None + member.digest = None + + variation.digest = None + digest = ga4gh_identify(variation, in_place="always") + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 + + return digest + + +def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: + """Normalize *allele* and stamp it with a freshly computed GA4GH digest. + + Pairs the finalize steps every VRS allele construction path needs. + Routing identification through :func:`identify_allele` (rather than + ``ga4gh_identify`` directly) is the invariant that protects against the + Merkle-tree's stale-digest behavior after mutation -- so any allele + construction site that bypasses this helper risks reintroducing the + stale-digest bug. + + Normalization can leave an indel as a ``ReferenceLengthExpression``; this coerces + it to a ``LiteralSequenceExpression`` so the result matches dcd_mapping's + authoritative alleles (``vrs_map._rle_to_lse``), which always store LSE. Without + the coercion the same biological indel hashes to two digests -- RLE here, LSE from + the mapper -- so reverse translation's regenerated genomic form fails to dedup + against the authoritative row and a duplicate allele is linked. + """ + allele = normalize(allele, data_proxy=data_proxy) + if isinstance(allele.state, ReferenceLengthExpression): + # Normalization yields an inlined SequenceLocation here, never an IRI reference. + assert isinstance(allele.location, SequenceLocation) + allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) + + allele.id = identify_allele(allele) + return allele + + +def _rle_to_lse( + rle: ReferenceLengthExpression, location: SequenceLocation, data_proxy: Any +) -> LiteralSequenceExpression: + """Coerce a ReferenceLengthExpression to an equivalent LiteralSequenceExpression. + + Mirrors ``dcd_mapping:vrs_map.py::_rle_to_lse`` byte-for-byte so an allele built here + hashes identically to the mapper's authoritative allele for the same variant. Derives + the literal sequence by tiling the repeat subunit out to ``rle.length``. + """ + # A normalized indel location has an inlined SequenceReference and integer bounds; + # the IRI-reference and Range branches of these unions should never reach this helper. + assert isinstance(location.sequenceReference, SequenceReference) + assert isinstance(location.start, int) + assert isinstance(rle.length, int) + + sequence_id = location.sequenceReference.refgetAccession + start = location.start + end = start + rle.repeatSubunitLength + subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) + c = cycle(subsequence) + derived_sequence = "".join(next(c) for _ in range(rle.length)) + return LiteralSequenceExpression(sequence=derived_sequence) diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index f23da015b..5b1a4cda8 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -20,6 +20,16 @@ def annotation_pipeline_job_definitions( [(upstream_mapping_key, DependencyType.SUCCESS_REQUIRED)] if upstream_mapping_key else [] ) return [ + { + "key": "reverse_translate_variants_for_score_set", + "function": "reverse_translate_variants_for_score_set", + "type": JobType.MAPPED_VARIANT_ANNOTATION, + "params": { + "correlation_id": None, # Required param to be filled in at runtime + "score_set_id": None, # Required param to be filled in at runtime + }, + "dependencies": mapping_dep, + }, { "key": "submit_score_set_mappings_to_car", "function": "submit_score_set_mappings_to_car", @@ -29,7 +39,7 @@ def annotation_pipeline_job_definitions( "score_set_id": None, # Required param to be filled in at runtime "updater_id": None, # Required param to be filled in at runtime }, - "dependencies": mapping_dep, + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, { "key": "warm_clingen_cache", @@ -86,16 +96,6 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_hgvs_for_score_set", - "function": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, # VEP annotation is intentionally not scheduled: its consequences came from VEP's top-level # most_severe_consequence, the worst call across all overlapping transcripts rather than the # one the variant was mapped to. @@ -110,16 +110,6 @@ def annotation_pipeline_job_definitions( # }, # "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], # }, - { - "key": "populate_variant_translations_for_score_set", - "function": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, ] diff --git a/src/mavedb/lib/workflow/kickoff.py b/src/mavedb/lib/workflow/kickoff.py new file mode 100644 index 000000000..94feec0bf --- /dev/null +++ b/src/mavedb/lib/workflow/kickoff.py @@ -0,0 +1,96 @@ +"""Create and enqueue a named pipeline for a single score set. + +The one shared kickoff seam: build the ``Pipeline`` / ``JobRun`` / ``JobDependency`` records via +:class:`PipelineFactory`, then enqueue the ``start_pipeline`` entrypoint in ARQ. Both the ad-hoc +``run_pipeline`` script and the batch ``enrich_backfilled_score_sets`` driver go through here so the +enqueue contract (entrypoint id + :func:`arq_job_id`) lives in exactly one place. +""" + +from __future__ import annotations + +import datetime +import logging +from typing import Any, Optional + +from arq.connections import ArqRedis +from arq.jobs import Job +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import correlation_id_for_context, logging_context, save_to_logging_context +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import arq_job_id + +logger = logging.getLogger(__name__) + + +async def enqueue_pipeline_for_score_set( + db: Session, + redis: ArqRedis, + *, + pipeline_name: str, + score_set: ScoreSet, + user: User, + extra_params: Optional[dict[str, Any]] = None, +) -> tuple[Pipeline, Optional[Job]]: + """Create the pipeline records for ``score_set`` and enqueue its ``start_pipeline`` entrypoint. + + Returns the created :class:`Pipeline` and the enqueued ARQ :class:`Job` (``None`` when ARQ + deduplicated an already-queued job with the same id). Raises ``KeyError`` for an unknown + ``pipeline_name`` and ``ValueError`` for an invalid pipeline definition, surfaced from + :meth:`PipelineFactory.create_pipeline`. If the ARQ enqueue itself raises, the just-created + pipeline records are discarded (via :meth:`PipelineFactory.discard_pipeline`) before the + exception is re-raised, so a failed enqueue never leaves an orphaned pipeline behind. + """ + correlation_id = ( + correlation_id_for_context() + or f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + ) + params: dict[str, Any] = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + if extra_params: + params.update(extra_params) + + factory = PipelineFactory(session=db) + pipeline, entrypoint = factory.create_pipeline( + pipeline_name=pipeline_name, creating_user=user, pipeline_params=params + ) + job_id = arq_job_id(entrypoint) + + try: + job = await redis.enqueue_job(entrypoint.job_function, entrypoint.id, _job_id=job_id) + except Exception: + logger.error( + "Failed to enqueue pipeline %s for score set %s; discarding pipeline %s", + pipeline_name, + score_set.urn, + pipeline.id, + extra=logging_context(), + ) + factory.discard_pipeline(pipeline) + raise + + save_to_logging_context({"pipeline_id": pipeline.id, "job_id": job_id}) + + if not job: + logger.info( + "Pipeline %s for score set %s already enqueued (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + else: + logger.info( + "Enqueued pipeline %s for score set %s (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + return pipeline, job diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..1cec88e97 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -1,8 +1,10 @@ __all__ = [ "access_key", "acmg_classification", + "allele", "collection", "clinical_control", + "clinvar_allele_link", "controlled_keyword", "doi_identifier", "ensembl_identifier", @@ -10,12 +12,15 @@ "experiment", "experiment_set", "genome_identifier", + "gnomad_allele_link", "gnomad_variant", "job_dependency", "job_run", "legacy_keyword", "license", "mapped_variant", + "mapping_record", + "mapping_record_allele", "pipeline", "publication_identifier", "published_variant", @@ -37,6 +42,9 @@ "uniprot_offset", "user", "variant_annotation_status", + "annotation_event", + "annotation_event_view", "variant", "variant_translation", + "vep_allele_consequence", ] diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py new file mode 100644 index 000000000..67eadd519 --- /dev/null +++ b/src/mavedb/models/allele.py @@ -0,0 +1,63 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.lib.hgvs import extract_accession + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + + +class Allele(Base): + __tablename__ = "alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + + vrs_digest: Mapped[str] = Column(String, nullable=False) + level = Column(String(length=16), nullable=False) + + hgvs_g = Column(String, nullable=True) + hgvs_c = Column(String, nullable=True) + hgvs_p = Column(String, nullable=True) + + clingen_allele_id = Column(String, nullable=True) + post_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + created_at = Column(Date, nullable=False, default=date.today) + updated_at = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the populated HGVS column (derived, not stored). + + Exactly one of hgvs_g/hgvs_c/hgvs_p is populated per allele, and the transcript is that + string's accession. Derived rather than stored so it cannot drift from the HGVS column + it duplicates. + """ + return extract_accession(self.hgvs_g or self.hgvs_c or self.hgvs_p or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(func.coalesce(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p), ":", 1) + + mapping_record_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="allele", + ) + + # Annotation links (VEP, gnomAD, ClinVar) deliberately carry no reverse collection here — they are + # one-directional annotation->Allele, navigated set-wise from the link tables, not from an Allele + # instance. Keep new annotation links one-directional unless a read path needs the navigation. + + __table_args__ = ( + UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + Index("ix_alleles_vrs_digest", "vrs_digest"), + Index("ix_alleles_level", "level"), + Index("ix_alleles_clingen_allele_id", "clingen_allele_id"), + ) diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py new file mode 100644 index 000000000..7b5f69b7f --- /dev/null +++ b/src/mavedb/models/annotation_event.py @@ -0,0 +1,140 @@ +""" +SQLAlchemy model for the annotation event log. + +One append-only log spanning the whole pipeline (mapping, reverse translation, +annotation). Its subjects are exactly two persistent entities — ``Variant`` and +``Allele`` — selected by ``annotation_type`` via the polymorphic-subject CHECK. +Everything else the pipeline touches (``MappingRecord``, ``MappingRecordAllele``, +the external value tables) is a vehicle or resolution path, never a status subject. + +This is deliberately **not** a ``ValidTime`` table. A SCD-2 state table is a lossy +projection of an event log — it discards the skip/reconfirm/no-op events that are +the audit point. "Current" is derived (``DISTINCT ON (subject, annotation_type) +… id DESC``), never a stored ``current`` flag. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, Optional + +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason + +if TYPE_CHECKING: + from mavedb.models.allele import Allele + from mavedb.models.job_run import JobRun + from mavedb.models.score_set import ScoreSet + from mavedb.models.variant import Variant + +VARIANT_SUBJECT_TYPES = ( + AnnotationType.VRS_MAPPING.value, + AnnotationType.CROSS_LEVEL_TRANSLATION.value, + AnnotationType.VARIANT_TRANSLATION.value, + AnnotationType.LDH_SUBMISSION.value, +) +"""annotation_type values whose subject is the variant (variant_id set, allele_id null)""" + +ALLELE_SUBJECT_TYPES = ( + AnnotationType.CLINGEN_ALLELE_ID.value, + AnnotationType.GNOMAD_ALLELE_FREQUENCY.value, + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE.value, + AnnotationType.CLINVAR_CONTROL.value, + AnnotationType.MAPPED_HGVS.value, +) +"""annotation_type values whose subject is the allele (allele_id set, variant_id null)""" + + +def _sql_in_list(values: tuple[str, ...]) -> str: + return ", ".join(f"'{v}'" for v in values) + + +class AnnotationEvent(Base): + """An append-only event recording the *status* (not value) of one pipeline + observation about a ``Variant`` or an ``Allele``. + + The value (frequency, consequence, CAID, control) lives in the domain + ValidTime tables; this log records disposition + why + when. Reading the + domain tables alone cannot distinguish confirmed-absence from never-checked — + that gap is the whole reason the log exists. + + NOTE: JSONB ``event_metadata`` is tracked as a mutable object via MutableDict, + which only catches top-level mutations. Mutating a nested object + requires ``flag_modified(instance, "event_metadata")``. + """ + + __tablename__ = "annotation_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + annotation_type: Mapped[AnnotationType] = mapped_column(String(50), nullable=False) + + # Exactly one is set, per ck_annotation_event_subject. + variant_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("variants.id", ondelete="RESTRICT"), nullable=True + ) + allele_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=True + ) + + disposition: Mapped[Disposition] = mapped_column(String(50), nullable=False) + + # Domain-specific code reusing in-code vocabularies (EventReason, plus MappingOutcome and RT + # skip_category for those two jobs); disposition is the public axis. + reason: Mapped[EventReason] = mapped_column(String(50), nullable=False) + + # gnomAD db_version / Ensembl release / ClinVar release / mapper version. + source_version: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + # DB column is "metadata"; the attribute avoids the reserved Declarative name. + event_metadata: Mapped[Optional[Dict[str, Any]]] = mapped_column( + "metadata", MutableDict.as_mutable(JSONB), nullable=True + ) + + job_run_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True + ) + score_set_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("scoresets.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + # One-directional (no back-ref) — match the annotation-link convention. + variant: Mapped[Optional["Variant"]] = relationship("Variant") + allele: Mapped[Optional["Allele"]] = relationship("Allele") + job_run: Mapped[Optional["JobRun"]] = relationship("JobRun") + score_set: Mapped[Optional["ScoreSet"]] = relationship("ScoreSet") + + __table_args__ = ( + # Polymorphic subject: the type picks exactly one subject column. + CheckConstraint( + f"(annotation_type IN ({_sql_in_list(VARIANT_SUBJECT_TYPES)}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_sql_in_list(ALLELE_SUBJECT_TYPES)}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + # latest-per-allele / latest-per-variant (the DISTINCT ON … id DESC projections) + Index("ix_annotation_event_allele_type_id", "allele_id", "annotation_type", text("id DESC")), + Index("ix_annotation_event_variant_type_id", "variant_id", "annotation_type", text("id DESC")), + # version-keyed skip + Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), + # audit by run + Index("ix_annotation_event_job_run_id", "job_run_id"), + # backs the score-set ON DELETE SET NULL cascade and score-set-scoped audit queries + Index("ix_annotation_event_score_set_id", "score_set_id"), + ) + + def __repr__(self) -> str: + subject = f"variant_id={self.variant_id}" if self.variant_id is not None else f"allele_id={self.allele_id}" + return ( + f"" + ) diff --git a/src/mavedb/models/annotation_event_view.py b/src/mavedb/models/annotation_event_view.py new file mode 100644 index 000000000..5c8547807 --- /dev/null +++ b/src/mavedb/models/annotation_event_view.py @@ -0,0 +1,96 @@ +"""``v_current_annotation_events`` — the current-state projection over the AnnotationEvent log. + +The log is append-only; "current" is derived, never stored. This view exposes the latest event per +``(subject, annotation_type)`` — one row per allele/variant per annotation type — so operators, BI, +and app code can read current status with a plain ``SELECT`` instead of re-deriving the +``DISTINCT ON`` everywhere. It mirrors the existing ``v_variant_annotations`` pattern. + +ClinVar is **multi-live**: an allele accumulates one live link per archival release, so its current +status is one row *per release*. The window partition folds ``source_version`` in **only** for +``clinvar_control`` (via a CASE that is constant-NULL for every other type, collapsing those back to +one row per subject+type). + +The subject is polymorphic — exactly one of ``variant_id`` / ``allele_id`` is set per row (enforced by +the log's CHECK), and the other is constant-NULL within a partition, so partitioning by both keys each +row to its real subject. + +There is deliberately **no ``score_set_id`` axis**: an allele-subject status (CAID, gnomAD, ClinVar, +VEP) is a *shared* allele-level fact, not a property of any one score set. A consumer that wants a +score set's annotation status resolves the score set's current alleles (and variants) through the live +mapping links — e.g. ``lib.clingen.alleles.get_alleles_for_score_set`` — and then looks those subjects +up here. Filtering by the *run's* score set would wrongly drop an allele last (re-)annotated by another +score set's run. The derived "current-for-variant" walk (resolving an allele-subject fact down to a +variant at the level a type keys on) is intentionally **not** built here — that is the deferred +consumer surface; see ``docs/design/allele-annotation-status.md``. +""" + +from sqlalchemy import case, func, select + +from mavedb.db.base import Base +from mavedb.db.view import view +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType + +signature = "v_current_annotation_events" + +# ClinVar is the lone multi-live type: split "current" by release. For every other type this is NULL, +# so the partition collapses to (allele_id, variant_id, annotation_type) — one current row per subject. +_clinvar_release_key = case( + (AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL.value, AnnotationEvent.source_version), + else_=None, +) + +_ranked = select( + AnnotationEvent.id.label("id"), + AnnotationEvent.annotation_type.label("annotation_type"), + AnnotationEvent.variant_id.label("variant_id"), + AnnotationEvent.allele_id.label("allele_id"), + AnnotationEvent.disposition.label("disposition"), + AnnotationEvent.reason.label("reason"), + AnnotationEvent.source_version.label("source_version"), + AnnotationEvent.event_metadata.label("event_metadata"), + AnnotationEvent.job_run_id.label("job_run_id"), + AnnotationEvent.created_at.label("created_at"), + func.row_number() + .over( + partition_by=[ + AnnotationEvent.allele_id, + AnnotationEvent.variant_id, + AnnotationEvent.annotation_type, + _clinvar_release_key, + ], + order_by=AnnotationEvent.id.desc(), + ) + .label("row_number"), +).subquery("ranked_annotation_events") + +definition = select( + _ranked.c.id, + _ranked.c.annotation_type, + _ranked.c.variant_id, + _ranked.c.allele_id, + _ranked.c.disposition, + _ranked.c.reason, + _ranked.c.source_version, + _ranked.c.event_metadata, + _ranked.c.job_run_id, + _ranked.c.created_at, +).where(_ranked.c.row_number == 1) + + +class CurrentAnnotationEventView(Base): + __table__ = view(signature, definition, materialized=False) + # Each surviving event id is unique across the view, so it is a valid mapping key for the + # otherwise-PK-less view (standard SQLAlchemy view-mapping idiom). + __mapper_args__ = {"primary_key": [__table__.c.id]} + + id = __table__.c.id + annotation_type = __table__.c.annotation_type + variant_id = __table__.c.variant_id + allele_id = __table__.c.allele_id + disposition = __table__.c.disposition + reason = __table__.c.reason + source_version = __table__.c.source_version + event_metadata = __table__.c.event_metadata + job_run_id = __table__.c.job_run_id + created_at = __table__.c.created_at diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 0b989cb22..388444a09 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, relationship @@ -8,33 +8,45 @@ from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table if TYPE_CHECKING: + from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant -class ClinicalControl(Base): - __tablename__ = "clinical_controls" +class ClinvarControl(Base): + __tablename__ = "clinvar_controls" __table_args__ = ( UniqueConstraint( - "db_name", "db_identifier", "db_version", name="uq_clinical_controls_db_name_identifier_version" + "db_name", "db_identifier", "db_version", name="uq_clinvar_controls_db_name_identifier_version" ), ) - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - gene_symbol = Column(String, nullable=False, index=True) + gene_symbol: Mapped[str] = Column(String, nullable=False, index=True) - clinical_significance = Column(String, nullable=False) - clinical_review_status = Column(String, nullable=False) + clinical_significance: Mapped[str] = Column(String, nullable=False) + clinical_review_status: Mapped[str] = Column(String, nullable=False) - db_name = Column(String, nullable=False, index=True) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False, index=True) + db_name: Mapped[str] = Column(String, nullable=False, index=True) + # ClinVar Allele ID (row level link). + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False, index=True) + + # ClinVar Variation ID (variation level link). + clinvar_variation_id: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen serving path: links to MappedVariant via the old association table (never written for new data). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=mapped_variants_clinical_controls_association_table, back_populates="clinical_controls", ) + + # New-model annotation links (one live link per allele per ClinVar release). + allele_links: Mapped[list["ClinvarAlleleLink"]] = relationship( + "ClinvarAlleleLink", + back_populates="clinvar_control", + ) diff --git a/src/mavedb/models/clinical_control_mapped_variant.py b/src/mavedb/models/clinical_control_mapped_variant.py index eabb7689a..b7fc5d84c 100644 --- a/src/mavedb/models/clinical_control_mapped_variant.py +++ b/src/mavedb/models/clinical_control_mapped_variant.py @@ -7,5 +7,5 @@ "mapped_variants_clinical_controls", Base.metadata, Column("mapped_variant_id", ForeignKey("mapped_variants.id"), primary_key=True), - Column("clinical_control_id", ForeignKey("clinical_controls.id"), primary_key=True), + Column("clinical_control_id", ForeignKey("clinvar_controls.id"), primary_key=True), ) diff --git a/src/mavedb/models/clinvar_allele_link.py b/src/mavedb/models/clinvar_allele_link.py new file mode 100644 index 000000000..941ee2f22 --- /dev/null +++ b/src/mavedb/models/clinvar_allele_link.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .clinical_control import ClinvarControl + + +class ClinvarAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a :class:`ClinvarControl` release. + + Replaces the frozen ``mapped_variants_clinical_controls`` association table for new-model writes. + A link is live while ``valid_to`` is NULL. Unlike gnomAD/VEP (one live result per allele), the partial + unique index is ``(allele_id, clinvar_control_id) WHERE valid_to IS NULL`` — **multi-live**: an allele + accumulates one live link per ClinVar release, because each release is a distinct, versioned + ``ClinvarControl`` assertion that stacks rather than supersedes. A link retires only on two theoretical + paths (archival data does not change): ClinVar drops the variant from a release (a re-run finds no data + for it), or the allele re-resolves to a *different* control within the same release — the job supersedes + that newest-wins to preserve one live link per (allele, release), since this index only enforces one live + link per (allele, control). + """ + + __tablename__ = "clinvar_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + clinvar_control_id: Mapped[int] = Column( + Integer, + ForeignKey("clinvar_controls.id", ondelete="RESTRICT"), + nullable=False, + ) + + # One-directional to Allele (no reverse collection there); back-ref on the ClinvarControl entity only. + allele: Mapped["Allele"] = relationship("Allele") + clinvar_control: Mapped["ClinvarControl"] = relationship("ClinvarControl", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_clinvar_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_clinvar_allele_links_clinvar_control_id", + "clinvar_control_id", + ), + # Multi-live: one live link per (allele, release). Each ClinVar release is a distinct + # ClinvarControl row, so different releases stack as independent live links rather than + # superseding. Only live rows participate in this constraint. + Index( + "uq_clinvar_allele_links_live", + "allele_id", + "clinvar_control_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/enums/annotation_layer.py b/src/mavedb/models/enums/annotation_layer.py deleted file mode 100644 index 2115f8e99..000000000 --- a/src/mavedb/models/enums/annotation_layer.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum - - -class AnnotationLayer(str, Enum): - """Annotation layer for a variant mapping result. - - Mirrors the ``AnnotationLayer`` enum produced by the dcd-mapping QC API. - Values use full names so they round-trip readably through the database - column; the dcd-mapping payload uses short single-character codes - (``p`` / ``c`` / ``g``) which the worker translates via :func:`from_wire`. - """ - - protein = "protein" - cdna = "cdna" - genomic = "genomic" - - @classmethod - def from_wire(cls, code: str) -> "AnnotationLayer": - """Translate a dcd-mapping single-character code into an ``AnnotationLayer``.""" - try: - return _WIRE_TO_LAYER[code] - except KeyError as exc: - raise ValueError(f"Unknown annotation_level wire code: {code!r}") from exc - - -# Module-level so it doesn't get mistaken for an enum member. -_WIRE_TO_LAYER: dict[str, AnnotationLayer] = { - "p": AnnotationLayer.protein, - "c": AnnotationLayer.cdna, - "g": AnnotationLayer.genomic, -} diff --git a/src/mavedb/models/enums/annotation_type.py b/src/mavedb/models/enums/annotation_type.py index b1595347b..a3bbd1693 100644 --- a/src/mavedb/models/enums/annotation_type.py +++ b/src/mavedb/models/enums/annotation_type.py @@ -3,6 +3,7 @@ class AnnotationType(str, Enum): VRS_MAPPING = "vrs_mapping" + CROSS_LEVEL_TRANSLATION = "cross_level_translation" CLINGEN_ALLELE_ID = "clingen_allele_id" MAPPED_HGVS = "mapped_hgvs" VARIANT_TRANSLATION = "variant_translation" diff --git a/src/mavedb/models/enums/disposition.py b/src/mavedb/models/enums/disposition.py new file mode 100644 index 000000000..94438f292 --- /dev/null +++ b/src/mavedb/models/enums/disposition.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Disposition(str, Enum): + """The stable, consumer-facing status axis of a variant event. + + Defined by *what the consumer may conclude* — not by the domain-specific + operation that produced it (that lives in ``reason``). + + - ``present`` — we hold the result / the step succeeded + - ``absent`` — the source or biology has nothing — an informative negative + - ``not_applicable`` — we could not ask — a pipeline/structural gap, not a statement about the source + - ``failed`` — errored, might retry; failure_category carries transient vs permanent + """ + + PRESENT = "present" + ABSENT = "absent" + NOT_APPLICABLE = "not_applicable" + FAILED = "failed" diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py new file mode 100644 index 000000000..2082a6163 --- /dev/null +++ b/src/mavedb/models/enums/event_reason.py @@ -0,0 +1,54 @@ +from enum import Enum + + +class EventReason(str, Enum): + """Pipeline-wide vocabulary for a variant event's ``reason`` — the single field that says + *what happened*, spanning present / absent / not_applicable / failed. + + Reasons are shared across jobs wherever they mean the same thing (the ``annotation_type`` + already says *which* job), and job-specific only where a case is genuinely unique. One job + contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses + ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the + external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + + ``MIGRATED`` is the one exception to "reason means what happened": it deliberately does not + distinguish created/preexisting/reconfirmed etc., because a one-off data migration reconstructing + state from a legacy source cannot know which of those actually happened historically — only + ``disposition`` (present/absent/failed) is knowable there. Using it elsewhere would misrepresent a + live job's real observation as a backfill guess. + """ + + # spans present / absent / failed — a reconstructed observation, not a live run + MIGRATED = "migrated" # state reconstructed from a world lacking now required context; no per-run history + + # present — we hold the result / the step succeeded + CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) + PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) + RECONFIRMED = "reconfirmed" # re-verified unchanged (gnomAD, CAR force) + SKIPPED = "skipped" # version-skip: already current at this source version (gnomAD, VEP) + SUPERSEDED = "superseded" # re-resolved within a release, newest wins (ClinVar) + SUBMITTED = "submitted" # LDH + TRANSLATED = "translated" # reverse translation + RECONFIRMATION_SKIPPED = "reconfirmation_skipped" # HGVS no longer buildable, existing CAID kept (CAR) + + # absent — the source or biology has nothing (informative negative) + NO_RECORD = "no_record" # source queried, returned nothing (gnomAD, ClinVar, VEP) + NO_CODING_TRANSCRIPT = "no_coding_transcript" # non-coding target has no protein consequence (RT) + + # not_applicable — we could not ask (structural gap) + NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) + NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) + MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + PROTEIN_LEVEL_ALLELE = ( + "protein_level_allele" # protein allele excluded from nucleotide-level annotation (ClinVar, gnomAD) + ) + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) + + # failed — errored + API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) + SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) + MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) + CAID_CONFLICT = "caid_conflict" # returned identifier conflicts with the stored one (CAR) + TRANSLATION_FAILED = "translation_failed" # all candidate HGVS failed translation (RT) + TRANSLATION_ERROR = "translation_error" # the translation engine errored (RT) + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" # protein-coding target with no resolvable transcript (RT) diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py new file mode 100644 index 000000000..62875f99f --- /dev/null +++ b/src/mavedb/models/enums/sequence_level.py @@ -0,0 +1,48 @@ +from enum import Enum + + +class SequenceLevel(str, Enum): + """The molecular sequence level of a variant representation: genomic DNA, coding DNA, or protein. + + A single, duty-neutral closed set reused across several columns that each carry a different + semantic meaning: the level a variant was *assayed* at (``assay_level``), the level dcd-mapping + *aligned* it at (``alignment_level``), and the level of a stored allele (``level``). + + Values use full names so they round-trip readably through the database column; the dcd-mapping + payload uses short single-character codes (``p`` / ``c`` / ``g``) which the worker translates via + :func:`from_wire`. + """ + + protein = "protein" + cdna = "cdna" + genomic = "genomic" + + @classmethod + def from_wire(cls, code: str) -> "SequenceLevel": + """Translate a dcd-mapping single-character code into a ``SequenceLevel``.""" + try: + return _WIRE_TO_LEVEL[code] + except KeyError as exc: + raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + + @classmethod + def nucleotide_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are nucleotide (genomic or coding).""" + return {cls.genomic, cls.cdna} + + @classmethod + def protein_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are protein (protein only).""" + return {cls.protein} + + +# Module-level so it doesn't get mistaken for an enum member. +_WIRE_TO_LEVEL: dict[str, SequenceLevel] = { + "p": SequenceLevel.protein, + "c": SequenceLevel.cdna, + "g": SequenceLevel.genomic, +} + + +NUCLEOTIDE_LEVELS = {level.value for level in SequenceLevel.nucleotide_levels()} +PROTEIN_LEVELS = {level.value for level in SequenceLevel.protein_levels()} diff --git a/src/mavedb/models/gnomad_allele_link.py b/src/mavedb/models/gnomad_allele_link.py new file mode 100644 index 000000000..75294ba66 --- /dev/null +++ b/src/mavedb/models/gnomad_allele_link.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .gnomad_variant import GnomADVariant + + +class GnomadAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a gnomAD variant. + + Replaces the frozen ``gnomad_variants_mapped_variants`` association table for new-model writes. + A link is live while ``valid_to`` is NULL; a gnomAD version bump retires the live row and inserts + a successor rather than deleting, so prior-version frequency links remain queryable point-in-time. + The partial unique index enforces **a single live link per allele** — gnomAD frequency is one + current value, so a new version supersedes the old (unlike ClinVar, which keeps one live link per + release). This matches the VEP consequence shape, not the ClinVar control shape. + """ + + __tablename__ = "gnomad_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + gnomad_variant_id: Mapped[int] = Column( + Integer, + ForeignKey("gnomad_variants.id", ondelete="RESTRICT"), + nullable=False, + ) + + allele: Mapped["Allele"] = relationship("Allele") + gnomad_variant: Mapped["GnomADVariant"] = relationship("GnomADVariant", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_gnomad_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_variant_id", + ), + # At most one live link per allele. A version bump supersedes (retires the old, inserts the + # new) rather than accumulating per-version live links; superseded rows stay for point-in-time + # queries. Only the live row participates in this constraint. + Index( + "uq_gnomad_allele_links_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index 0f4a00a50..bb9a1faf6 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,37 +1,46 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional -from sqlalchemy import Column, Date, Integer, Float, String +from sqlalchemy import Column, Date, Float, Integer, String from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: + from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant class GnomADVariant(Base): __tablename__ = "gnomad_variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - db_name = Column(String, nullable=False) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False) + db_name: Mapped[str] = Column(String, nullable=False) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False) - allele_count = Column(Integer, nullable=False) - allele_number = Column(Integer, nullable=False) - allele_frequency = Column(Float, nullable=False) + allele_count: Mapped[int] = Column(Integer, nullable=False) + allele_number: Mapped[int] = Column(Integer, nullable=False) + allele_frequency: Mapped[float] = Column(Float, nullable=False) - faf95_max = Column(Float, nullable=True) - faf95_max_ancestry = Column(String, nullable=True) + faf95_max: Mapped[Optional[float]] = Column(Float, nullable=True) + faf95_max_ancestry: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen association to the old MappedVariant model — read by serving for existing data, never + # written for new score sets (which link through ``allele_links`` instead). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=gnomad_variants_mapped_variants_association_table, back_populates="gnomad_variants", ) + + # Valid-time links to deduplicated alleles (new-model writes). + allele_links: Mapped[list["GnomadAlleleLink"]] = relationship( + "GnomadAlleleLink", + back_populates="gnomad_variant", + ) diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index b35f1bbb4..10914f778 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -7,11 +7,11 @@ from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: - from .clinical_control import ClinicalControl + from .clinical_control import ClinvarControl from .gnomad_variant import GnomADVariant from .target_gene_mapping import TargetGeneMapping from .variant import Variant @@ -44,7 +44,7 @@ class MappedVariant(Base): # Per-mapping QC annotations from dcd-mapping ScoreAnnotation. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) @@ -61,8 +61,8 @@ class MappedVariant(Base): hgvs_c = Column(String, nullable=True) hgvs_p = Column(String, nullable=True) - clinical_controls: Mapped[list["ClinicalControl"]] = relationship( - "ClinicalControl", + clinical_controls: Mapped[list["ClinvarControl"]] = relationship( + "ClinvarControl", secondary=mapped_variants_clinical_controls_association_table, back_populates="mapped_variants", ) diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py new file mode 100644 index 000000000..dda9ef16e --- /dev/null +++ b/src/mavedb/models/mapping_record.py @@ -0,0 +1,112 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime +from mavedb.lib.hgvs import extract_accession +from mavedb.models.enums.sequence_level import SequenceLevel + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + from .target_gene_mapping import TargetGeneMapping + from .variant import Variant + + +class MappingRecord(ValidTime, Base): + __tablename__ = "mapping_records" + + id: Mapped[int] = Column(Integer, primary_key=True) + + variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False) + variant: Mapped["Variant"] = relationship("Variant", back_populates="mapping_records") + + # Digest of the pre-mapped (assayed-level) VRS representation, indexed for + # cross-score-set dedup lookups. + vrs_digest = Column(String, nullable=True) + pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + # Level at which the variant was *assayed* — distinct from alignment_level (QC). Same closed + # SequenceLevel set as its sibling below. The prod CHECK (ck_mapping_records_assay_level_valid) + # predates this column being typed as an Enum; native_enum=False keeps it a VARCHAR(16), so + # declaring the Enum here needs no migration — it self-documents the model and gives the + # metadata-built (test) schema the same guard. + assay_level: Mapped[SequenceLevel] = Column( + # Distinct `name` so the generated CHECK doesn't collide with alignment_level's in this same + # table (both default to "sequencelevel", which Postgres rejects as a duplicate per table). + Enum( + SequenceLevel, + name="assay_level", + create_constraint=True, + length=16, + native_enum=False, + validate_strings=True, + ), + nullable=False, + ) + hgvs_assay_level = Column(String, nullable=True) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the assay-level HGVS (derived, not stored). + + Derived rather than stored so it cannot drift from hgvs_assay_level, which already + carries the accession. + """ + return extract_accession(self.hgvs_assay_level or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(cls.hgvs_assay_level, ":", 1) + + mapping_api_version = Column(String, nullable=False) + # Domain data: the date the mapping was performed (from the dcd-mapping run), surfaced to + # users — distinct from valid_from, which is when this version became the live record. + mapped_date = Column(Date, nullable=False, default=date.today) + vrs_version = Column(String, nullable=True) + + # valid_from / valid_to and the derived `current` come from ValidTime: a re-map retires the + # prior live record (closes its valid_to) and inserts a new live one, so mapping history is + # retained and point-in-time queries are a single predicate. + + # Per-mapping QC fields from dcd-mapping. + alignment_level = Column( + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), + nullable=True, + ) + at_mismatched_locus = Column(Boolean, nullable=True) + near_gap = Column(Boolean, nullable=True) + + target_gene_mapping_id = Column(Integer, ForeignKey("target_gene_mappings.id"), index=True, nullable=True) + target_gene_mapping: Mapped[Optional["TargetGeneMapping"]] = relationship( + "TargetGeneMapping", back_populates="mapping_records" + ) + + allele_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + # Retiring a record retires its live allele links too (both the authoritative link and any + # derived links a reverse-translation run attached). See ValidTime.__retire_cascade__. + __retire_cascade__ = ("allele_links",) + + __table_args__ = ( + Index("ix_mapping_records_vrs_digest", "vrs_digest"), + Index("ix_mapping_records_variant_id", "variant_id"), + # At most one live mapping record per variant — promotes to the database the invariant the + # mapping job enforces in app code (it retires the prior live record before inserting a new + # one). Superseded versions remain with a closed valid_to for point-in-time queries. + Index( + "uq_mapping_records_current", + "variant_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py new file mode 100644 index 000000000..ce57f259d --- /dev/null +++ b/src/mavedb/models/mapping_record_allele.py @@ -0,0 +1,67 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .mapping_record import MappingRecord + + +class MappingRecordAllele(ValidTime, Base): + __tablename__ = "mapping_record_alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column( + Integer, + ForeignKey("mapping_records.id", ondelete="CASCADE"), + nullable=False, + ) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + # True for the assay's actual measured allele (from vrs_map); False for translator-derived + # alleles. Lives on the link, not Allele, since the same allele can be authoritative for one + # mapping record and derived for another. + is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) + # Pairs a link with its coding/genomic counterpart within one mapping record (set by the + # reverse-translation job). NULL for unpaired links. Local to this record, not a cross-record + # identity — see the RT job for group-assignment logic. + projection_group = Column(Integer, nullable=True) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") + allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") + + __table_args__ = ( + Index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_id", + ), + Index( + "ix_mapping_record_alleles_allele_id", + "allele_id", + ), + # At most one live link per (mapping_record, allele); retired links (valid_to set) are + # exempt so history is preserved. + Index( + "uq_mapping_record_alleles_live", + "mapping_record_id", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + # At most one live authoritative link per mapping record. lib/score_set_variants relies on + # this 1:1 invariant (inner join on the authoritative link); a second one would silently + # duplicate the variant and double-count its score. + Index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_id", + unique=True, + postgresql_where=text("is_authoritative AND valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/published_variant.py b/src/mavedb/models/published_variant.py index eff749bd7..503d3fa30 100644 --- a/src/mavedb/models/published_variant.py +++ b/src/mavedb/models/published_variant.py @@ -1,25 +1,29 @@ -from sqlalchemy import select, join +from sqlalchemy import join, select from mavedb.db.view import MaterializedView, view - +from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.mapped_variant import MappedVariant - signature = "published_variants_materialized_view" + +# One row per (published variant, mapping-record version). The LEFT OUTER JOIN to mapping_records +# carries NO liveness filter on purpose: every record version (live and superseded) is retained so +# the statistics layer can count either the current mappings or the full history via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). Unmapped variants stay in the view with a +# NULL ``mapping_record_id``. definition = ( select( Variant.id.label("variant_id"), Variant.urn.label("variant_urn"), - MappedVariant.id.label("mapped_variant_id"), + MappingRecord.id.label("mapping_record_id"), ScoreSet.id.label("score_set_id"), ScoreSet.urn.label("score_set_urn"), ScoreSet.published_date.label("published_date"), - MappedVariant.current.label("current_mapped_variant"), + MappingRecord.valid_to.is_(None).label("current_mapping_record"), ) .select_from( - join(Variant, MappedVariant, Variant.id == MappedVariant.variant_id, isouter=True).join( + join(Variant, MappingRecord, Variant.id == MappingRecord.variant_id, isouter=True).join( ScoreSet, ScoreSet.id == Variant.score_set_id ) ) @@ -38,8 +42,8 @@ class PublishedVariantsMV(MaterializedView): variant_id = __table__.c.variant_id variant_urn = __table__.c.variant_urn - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id score_set_id = __table__.c.score_set_id score_set_urn = __table__.c.score_set_urn published_date = __table__.c.published_date - current_mapped_variant = __table__.c.current_mapped_variant + current_mapping_record = __table__.c.current_mapping_record diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 38ce1f286..1941cb863 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -25,7 +25,7 @@ class ScoreCalibration(Base): __tablename__ = "score_calibrations" # TODO#544: Add a partial unique index to enforce only one primary calibration per score set. - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), nullable=True, default=generate_calibration_urn, unique=True, index=True) score_set_id = Column(Integer, ForeignKey("scoresets.id"), nullable=False) @@ -33,7 +33,7 @@ class ScoreCalibration(Base): title = Column(String, nullable=False) research_use_only = Column(Boolean, nullable=False, default=False) - primary = Column(Boolean, nullable=False, default=False) + primary: Mapped[bool] = Column(Boolean, nullable=False, default=False) investigator_provided: Mapped[bool] = Column(Boolean, nullable=False, default=False) private = Column(Boolean, nullable=False, default=True) notes = Column(String, nullable=True) diff --git a/src/mavedb/models/score_set.py b/src/mavedb/models/score_set.py index ccdcb73d5..b61704ffe 100644 --- a/src/mavedb/models/score_set.py +++ b/src/mavedb/models/score_set.py @@ -74,7 +74,7 @@ class ScoreSet(Base): __tablename__ = "scoresets" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) # TODO(#372) urn = Column(String(64), default=generate_temp_urn, index=True, nullable=True, unique=True) diff --git a/src/mavedb/models/target_gene.py b/src/mavedb/models/target_gene.py index 671a4380f..5419c3478 100644 --- a/src/mavedb/models/target_gene.py +++ b/src/mavedb/models/target_gene.py @@ -23,10 +23,10 @@ class TargetGene(Base): __tablename__ = "target_genes" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) name = Column(String, nullable=False) - category = Column( + category: Mapped[TargetCategory] = Column( Enum(TargetCategory, create_constraint=True, length=32, native_enum=False, validate_strings=True), nullable=False, ) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 417f249d4..5ead4a887 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -19,23 +19,24 @@ from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant + from mavedb.models.mapping_record import MappingRecord from mavedb.models.target_gene import TargetGene class TargetGeneMapping(Base): __tablename__ = "target_gene_mappings" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - target_gene_id = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) + target_gene_id: Mapped[int] = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=False, ) preferred = Column(Boolean, nullable=False, default=False, server_default="false") @@ -80,4 +81,9 @@ class TargetGeneMapping(Base): back_populates="target_gene_mapping", ) + mapping_records: Mapped[list["MappingRecord"]] = relationship( + "MappingRecord", + back_populates="target_gene_mapping", + ) + __table_args__ = (Index("ix_target_gene_mappings_target_alignment", "target_gene_id", "alignment_level"),) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 59b6e729c..dd46c430c 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -9,13 +9,14 @@ if TYPE_CHECKING: from .mapped_variant import MappedVariant + from .mapping_record import MappingRecord from .score_set import ScoreSet class Variant(Base): __tablename__ = "variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), index=True, nullable=True, unique=True) data = Column(JSONB, nullable=False) @@ -35,5 +36,9 @@ class Variant(Base): back_populates="variant", cascade="all, delete-orphan" ) + mapping_records: Mapped[List["MappingRecord"]] = relationship( + back_populates="variant", cascade="all, delete-orphan" + ) + # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. diff --git a/src/mavedb/models/variant_annotation_status.py b/src/mavedb/models/variant_annotation_status.py index f39c47a64..357d6a6e6 100644 --- a/src/mavedb/models/variant_annotation_status.py +++ b/src/mavedb/models/variant_annotation_status.py @@ -93,7 +93,7 @@ class VariantAnnotationStatus(Base): # FK index for job_run_id — needed for CASCADE deletes on job_runs Index("ix_variant_annotation_status_job_run_id", "job_run_id"), CheckConstraint( - "annotation_type IN ('vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", + "annotation_type IN ('vrs_mapping', 'cross_level_translation', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", name="ck_variant_annotation_type_valid", ), CheckConstraint( diff --git a/src/mavedb/models/variant_annotation_view.py b/src/mavedb/models/variant_annotation_view.py index c7c77d6f8..6b6e52963 100644 --- a/src/mavedb/models/variant_annotation_view.py +++ b/src/mavedb/models/variant_annotation_view.py @@ -2,13 +2,89 @@ from mavedb.db.base import Base from mavedb.db.view import view -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence signature = "v_variant_annotations" +# All allele-derived columns are correlated scalar subqueries keyed on the outer MappingRecord, so the +# outer query's FROM stays Variant / ScoreSet / MappingRecord / VariantAnnotationStatus and never +# references Allele / MappingRecordAllele directly. That keeps the grain at one row per +# (variant, annotation_type) — matching the legacy view — and, crucially, avoids ``aliased()`` at +# module import (which would force full mapper configuration before every model is registered, breaking +# a bare ``import`` of this module by Alembic). + + +def _auth_allele(column): + """Scalar subquery: ``column`` of the outer record's live **authoritative** (measured) allele. + + Source of the single-valued ``clingen_allele_id``. The authoritative link is 1:1 with the record + (the mapping job upholds one live authoritative link per record), so this resolves a single allele. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _auth_vep(column): + """Scalar subquery: ``column`` of the live VEP consequence on the record's authoritative allele.""" + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _record_hgvs(level: str, column): + """Scalar subquery: the canonical HGVS at ``level`` for the outer record. + + Each mapped level (genomic / cdna / protein) is a distinct deduplicated :class:`Allele` linked to + the record. This constructs a flat ``hgvs_g`` / ``hgvs_c`` / ``hgvs_p`` triple by pivoting the record's + live allele links by level — yielding ``NULL`` where the record has no allele at that level. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + Allele.level == level, + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +# Flat operator/CLI convenience view: one row per (variant, current annotation_type). definition = ( select( Variant.urn.label("variant_urn"), @@ -16,18 +92,17 @@ Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, - MappedVariant.id.label("mapped_variant_id"), - MappedVariant.clingen_allele_id, - MappedVariant.hgvs_assay_level, - MappedVariant.hgvs_g, - MappedVariant.hgvs_c, - MappedVariant.hgvs_p, - MappedVariant.vep_functional_consequence, - MappedVariant.vep_access_date, - MappedVariant.mapped_date, - MappedVariant.mapping_api_version, - MappedVariant.vrs_version, - MappedVariant.error_message.label("mapping_error"), + MappingRecord.id.label("mapping_record_id"), + _auth_allele(Allele.clingen_allele_id).label("clingen_allele_id"), + MappingRecord.hgvs_assay_level, + _record_hgvs("genomic", Allele.hgvs_g).label("hgvs_g"), + _record_hgvs("cdna", Allele.hgvs_c).label("hgvs_c"), + _record_hgvs("protein", Allele.hgvs_p).label("hgvs_p"), + _auth_vep(VepAlleleConsequence.functional_consequence).label("vep_functional_consequence"), + _auth_vep(VepAlleleConsequence.access_date).label("vep_access_date"), + MappingRecord.mapped_date, + MappingRecord.mapping_api_version, + MappingRecord.vrs_version, VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status.label("annotation_status"), VariantAnnotationStatus.failure_category, @@ -40,10 +115,9 @@ ) .select_from(Variant) .join(ScoreSet, ScoreSet.id == Variant.score_set_id) - .outerjoin( - MappedVariant, - and_(MappedVariant.variant_id == Variant.id, MappedVariant.current == True), # noqa: E712 - ) + # The variant's live mapping record. live-ness rides the ON clause (never a WHERE) so the outer + # join is preserved for unmapped variants — a WHERE would reject the null-record row. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) .outerjoin( VariantAnnotationStatus, and_(VariantAnnotationStatus.variant_id == Variant.id, VariantAnnotationStatus.current == True), # noqa: E712 @@ -59,7 +133,7 @@ class VariantAnnotationView(Base): hgvs_nt = __table__.c.hgvs_nt hgvs_pro = __table__.c.hgvs_pro hgvs_splice = __table__.c.hgvs_splice - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id clingen_allele_id = __table__.c.clingen_allele_id hgvs_assay_level = __table__.c.hgvs_assay_level hgvs_g = __table__.c.hgvs_g @@ -70,7 +144,6 @@ class VariantAnnotationView(Base): mapped_date = __table__.c.mapped_date mapping_api_version = __table__.c.mapping_api_version vrs_version = __table__.c.vrs_version - mapping_error = __table__.c.mapping_error annotation_type = __table__.c.annotation_type annotation_status = __table__.c.annotation_status failure_category = __table__.c.failure_category diff --git a/src/mavedb/models/variant_translation.py b/src/mavedb/models/variant_translation.py index a50d36ce2..36aa0dfdf 100644 --- a/src/mavedb/models/variant_translation.py +++ b/src/mavedb/models/variant_translation.py @@ -6,6 +6,10 @@ class VariantTranslation(Base): + """FROZEN (serving-only). Written by the retired populate_variant_translations_for_score_set job; + superseded by the reverse-translation allele equivalence space. Read for existing old-model data, + never written for new score sets, dropped at read-cutover. See lib/variant_translations.py.""" + __tablename__ = "variant_translations" aa_clingen_id = Column(String, nullable=False, primary_key=True) diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py new file mode 100644 index 000000000..4133a1231 --- /dev/null +++ b/src/mavedb/models/vep_allele_consequence.py @@ -0,0 +1,75 @@ +from datetime import date +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + + +class VepAlleleConsequence(ValidTime, Base): + """Valid-time VEP functional-consequence result for a deduplicated :class:`Allele`. + + Replaces the frozen ``vep_functional_consequence``/``vep_access_date`` columns on + ``MappedVariant`` for new-model writes (Step 2 of the annotation infrastructure migration, + docs/design/annotation-infrastructure-migration.md). A row is live while ``valid_to`` is NULL; + the partial unique index enforces **a single live consequence per allele** — VEP's most-severe + consequence is one current value, so a changed result supersedes the prior row rather than + accumulating. This matches the gnomAD link shape, not ClinVar's multi-live shape. + + ``source_version`` is the Ensembl release the consequence was resolved under (e.g. ``"116"``, + from ``/info/software``). An Ensembl release is coordinated — software + transcript set + + consequence vocabulary all bump together under one number — so this single value version-keys the + upstream result exactly like gnomAD's ``db_version``. The job skips re-querying any allele already + live at the current release. What it does **not** capture is our own ``VEP_CONSEQUENCES`` severity + ordering (the list we pick "most severe" from); a change to that is a manual ``force`` re-run, not + an automatic supersede. + + Supersede is deliberately **value-keyed, not version-keyed** (the one divergence from gnomAD): a VEP + consequence is categorical and usually identical across releases, so superseding on every release + bump would churn history every quarter with rows recording "still missense, still missense." Instead + a new release that resolves the *same* consequence advances ``source_version``/``access_date`` in + place — no supersede — and only a *changed* consequence retires the old row and inserts a successor. + The trade-off: the live row's ``source_version`` is the latest release that confirmed the value, not + the release it first appeared; acceptable because it describes the currently-held value's + provenance, not when it became true. ``access_date`` is retained as a human-facing "last confirmed" + audit stamp; it is no longer load-bearing for the skip. + + ``functional_consequence`` is nullable to leave room for a future negative cache (NULL = "VEP ran + and found nothing"); the current job writes only non-null consequences and re-queries no-result + alleles each run, mirroring gnomAD's no-match handling. + """ + + __tablename__ = "vep_allele_consequences" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + functional_consequence: Mapped[Optional[str]] = Column(String, nullable=True) + source_version: Mapped[str] = Column(String, nullable=False) + access_date: Mapped[date] = Column(Date, nullable=False) + + allele: Mapped["Allele"] = relationship("Allele") + + __table_args__ = ( + Index( + "ix_vep_allele_consequences_allele_id", + "allele_id", + ), + # At most one live consequence per allele. A changed result supersedes (retires the old, + # inserts the new) rather than accumulating; superseded rows stay for point-in-time queries. + # Only the live row participates in this constraint. + Index( + "uq_vep_allele_consequences_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py new file mode 100644 index 000000000..470765063 --- /dev/null +++ b/src/mavedb/routers/alleles.py @@ -0,0 +1,129 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Path, Query, Response +from fastapi.exceptions import HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.convertors import Convertor, register_url_convertor + +from mavedb import deps +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.routers.shared import ( + ACCESS_CONTROL_ERROR_RESPONSES, + PUBLIC_ERROR_RESPONSES, + ROUTER_BASE_PREFIX, +) +from mavedb.view_models.allele_detail import AlleleDetail + +TAG_NAME = "Alleles" + +logger = logging.getLogger(__name__) + + +# One resource, three identifier forms — a GA4GH VRS digest (one allele), a ClinGen nucleotide id (CAID, +# the nt-canonical change) or protein id (PAID). Rather than split the path or add a query param, overload +# a single `/alleles/{identifier}` with a custom Starlette convertor (the same pattern the publication +# router uses for DOI/PubMed/…). The convertor matches any of the three forms; the handler dispatches by +# form. In the OpenAPI spec this is still just a string path parameter. +class AlleleIdentifierConverter(Convertor): + # A GA4GH IR (ga4gh:.<32 base64url>) OR a ClinGen allele id (CA…/PA… + digits). + # NOTE: Multivariant CAIDs (CA…/PA… + digits + `,` + CA…/PA… + digits) are intentionally excluded. + regex = r"(?:ga4gh:[^.]+\.[0-9A-Za-z_\-]{32})|(?:[CP]A[0-9]+)" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return str(value) + + +register_url_convertor("allele_identifier", AlleleIdentifierConverter()) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Retrieve deduplicated alleles — identity, cross-layer equivalence class, and " + "external annotations — by VRS digest or ClinGen allele id (CAID / PAID).", +} + + +def _representative(alleles: list[Allele]) -> Allele: + """The allele whose identity fills the flat anchor fields for a ClinGen-id fetch. A CAID names one nt + change in up to two frames (genomic + coding) — prefer the coding frame, then genomic; a PAID matches + the single protein allele, which falls through to ``alleles[0]``.""" + by_level = {a.level: a for a in alleles} + return by_level.get(SequenceLevel.cdna.value) or by_level.get(SequenceLevel.genomic.value) or alleles[0] + + +@router.get( + "/alleles/{identifier:allele_identifier}", + status_code=200, + response_model=AlleleDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="Fetch allele detail by VRS digest, CAID, or PAID", +) +def get_allele( + *, + response: Response, + identifier: str = Path( + description="A GA4GH VRS digest (one allele), or a ClinGen allele id: a nucleotide CAID " + "(the nt-canonical change, its genomic + coding frames) or a protein PAID.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (equivalence-class membership + VEP/gnomAD/ClinVar " + "annotations) as it stood at this instant. ISO 8601, ideally timezone-aware. The focus " + "allele's own identity is content-addressed and immutable, so it is unaffected. Defaults to " + "current." + ), + ), + db: Session = Depends(deps.get_db), +): + """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. + + The allele-grain counterpart of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, + ClinGen id, spec-pure VRS) plus the cross-layer equivalence class (each member labelled relative to + the focus) and a digest-keyed annotation map. The ``identifier`` may be: + + - a **VRS digest** (``ga4gh:VA.…``) — focuses that one allele. + - a **CAID** (``CA…``) — the nt-canonical change; The coding frame is the preferential focus, + falling back to the genomic frame if no coding frame exists. + - a **PAID** (``PA…``) — the protein change; the protein allele is focused and its nucleotide + equivalents surface as reverse-translation candidates. + + This is a **public molecular resource**. It carries no score-set-level information. No scores, + classifications, measurements, or version standing. Only the allele's own identity, its cross-layer + equivalence class, and public reference annotations (VEP / gnomAD / ClinVar). + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + if identifier.startswith("ga4gh:"): + allele = db.scalar(select(Allele).where(Allele.vrs_digest == identifier)) + if allele is None: + raise HTTPException(status_code=404, detail=f"allele with VRS digest '{identifier}' not found") + return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of) + + # A ClinGen allele id (CAID or PAID): resolve to its allele(s) — the nt-canonical change's genomic + + # coding frames for a CAID, or the single protein allele for a PAID — and focus all of them. + matches = list(db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all()) + if not matches: + raise HTTPException(status_code=404, detail=f"no allele with ClinGen allele id '{identifier}' found") + + focus_digests = {a.vrs_digest for a in matches if a.vrs_digest is not None} + return get_allele_detail(db, _representative(matches), focus_digests=focus_digests, as_of=as_of) diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py new file mode 100644 index 000000000..b3b626c02 --- /dev/null +++ b/src/mavedb/routers/clingen_alleles.py @@ -0,0 +1,85 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Query, Response +from sqlalchemy.orm import Session + +from mavedb import deps +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.authentication import get_current_user +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData +from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.view_models.allele_measurement import AlleleMeasurement + +TAG_NAME = "ClinGen alleles" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/clingen-alleles", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "The ClinGen-allele-centric variant page: measurements across a variant's equivalence class.", +} + + +@router.get( + "/{clingen_allele_id}/measurements", + status_code=200, + response_model=list[AlleleMeasurement], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="List measurements for a ClinGen allele's equivalence class", +) +def get_clingen_allele_measurements( + *, + clingen_allele_id: str, + response: Response, + include_superseded: bool = Query( + default=False, + description=( + "Include measurements from superseded score-set versions. Default false — superseded " + "measurements are a deliberate power-user / citation path, never surfaced by discovery." + ), + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the equivalence class (which mapping records / allele links are live) as it " + "stood at this instant. ISO 8601, ideally timezone-aware. Scores and classifications are " + "as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """List every measurement whose cross-layer equivalence class touches this ClinGen allele (a ``CA`` or + ``PA``) — the direct measurements assayed at this change plus the reverse-translation-related ones, + each labeled by its assayed level and relationship. This is the ClinGen-allele-centric variant page's + entrypoint. A private score set's measurement is never included; its inline classification is withheld + where the calibration is unreadable while the measurement still shows. + """ + save_to_logging_context( + { + "requested_resource": clingen_allele_id, + "as_of": as_of, + "include_superseded": include_superseded, + } + ) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + return get_allele_measurements( + db, + clingen_allele_id, + user_data=user_data, + include_superseded=include_superseded, + as_of=as_of, + ) diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index debaebcb0..7a901d545 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -227,6 +227,13 @@ def get_experiment_score_sets( for fs in filtered_score_sets: enriched_experiment = enrich_experiment_with_num_score_sets(fs.experiment, user_data) visible_calibration_ids = {calibration.id for calibration in viewer.visible(fs.score_calibrations)} + # A superseded score set reaches this list *because* its successor failed the caller's READ check + # (see find_superseded_score_set_tail), so serializing that successor's urn and title would name + # the very score set the caller was found not to be entitled to. + # TODO(#808): this duplicates score_sets._score_set_response; the two have already drifted once. + superseding_is_visible = fs.superseding_score_set is not None and ( + has_permission(user_data, fs.superseding_score_set, Action.READ).permitted + ) validated_item = score_set.ScoreSet.model_validate(fs) response_item = validated_item.copy( update={ @@ -236,6 +243,7 @@ def get_experiment_score_sets( for calibration in (validated_item.score_calibrations or []) if calibration.id in visible_calibration_ids ], + "superseding_score_set": validated_item.superseding_score_set if superseding_is_visible else None, } ) enriched_score_sets.append(response_item) diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index f6bfb7f97..a42fa2503 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -1,232 +1,81 @@ import logging -from typing import Annotated, Any, Optional +from typing import Annotated +from urllib.parse import quote -from fastapi import APIRouter, Depends, Path -from fastapi.exceptions import HTTPException +from fastapi import APIRouter, Path, Request +from fastapi.responses import RedirectResponse from ga4gh.core.identifiers import GA4GH_IR_REGEXP -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session -from mavedb import deps -from mavedb.lib.annotation.annotate import ( - variant_functional_impact_statement, - variant_pathogenicity_statement, - variant_study_result, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user, get_principal from mavedb.lib.logging import LoggedRoute -from mavedb.lib.permissions.principal import Principal -from mavedb.lib.logging.context import ( - logging_context, - save_to_logging_context, -) -from mavedb.lib.permissions import Action, assert_permission, has_permission -from mavedb.lib.types.authentication import UserData -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX -from mavedb.view_models import mapped_variant +from mavedb.routers.shared import ROUTER_BASE_PREFIX -TAG_NAME = "Mapped Variants" +TAG_NAME = "Variants" logger = logging.getLogger(__name__) - -async def fetch_mapped_variant_by_variant_urn(db: Session, user: Optional[UserData], urn: str) -> MappedVariant: - """ - We may combine this function back to show_mapped_variant if none of any new function call it. - Fetch one mapped variant by variant URN. - - :param db: An active database session. - :param urn: The variant URN. - :return: The mapped variant. - - :raises HTTPException: If the mapped variant is not found or if multiple variants are found. - """ - try: - item = ( - db.execute(select(MappedVariant).join(Variant).where(Variant.urn == urn, MappedVariant.current.is_(True))) - .scalars() - .one_or_none() - ) - except MultipleResultsFound: - logger.info( - msg="Could not fetch the requested mapped variant; Multiple such variants exist.", extra=logging_context() - ) - raise HTTPException(status_code=500, detail=f"Multiple variants with URN {urn} were found.") - - if not item: - logger.info( - msg="Could not fetch the requested mapped variant; No such mapped variants exist.", extra=logging_context() - ) - - raise HTTPException(status_code=404, detail=f"Mapped variant with URN {urn} not found") - - # Base mapped variant read permission on the score set in which it is contained. - assert_permission(user, item.variant.score_set, Action.READ) - return item - - router = APIRouter( prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", tags=[TAG_NAME], - responses={**PUBLIC_ERROR_RESPONSES}, route_class=LoggedRoute, ) -metadata = { - "name": TAG_NAME, - "description": "Retrieve mapped variants and their associated variant annotations.", -} + +def _redirect(request: Request, target: str) -> RedirectResponse: + url = f"{target}?{request.url.query}" if request.url.query else target + return RedirectResponse(url=url, status_code=301) @router.get( "/{urn}", - status_code=200, - response_model=mapped_variant.MappedVariantWithMappingDetails, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variant by URN", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}", ) -async def show_mapped_variant( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Any: - """ - Fetch a single mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - return await fetch_mapped_variant_by_variant_urn(db, user, urn) +def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") @router.get( "/{urn}/va/study-result", - status_code=200, - response_model=ExperimentalVariantFunctionalImpactStudyResult, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec StudyResult from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/study-result", ) -async def show_mapped_variant_study_result( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> ExperimentalVariantFunctionalImpactStudyResult: - """ - Construct a single VA-Spec StudyResult from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) +def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/study-result`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") - try: - return variant_study_result(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg=f"Could not construct a study result for mapped variant {urn}: {e}", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No study result exists for mapped variant {urn}: {e}") - -# TODO#416: For now, this route supports only one statement per mapped variant. Eventually, we should support the possibility of multiple statements. @router.get( "/{urn}/va/functional-statement", - status_code=200, - response_model=Statement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec Statement from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/functional-statement", ) -async def show_mapped_variant_functional_impact_statement( - *, - urn: str, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), - principal: Principal = Depends(get_principal), -) -> Statement: - """ - Construct a single VA-Spec Statement from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - functional_impact = variant_functional_impact_statement(mapped_variant, principal=principal) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, detail=f"No functional impact statement exists for mapped variant {urn}: {e}" - ) - - if not functional_impact: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant. Variant does not have sufficient evidence to evaluate its functional impact.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No functional impact statement exists for mapped variant {urn}. Variant does not have sufficient evidence to evaluate its functional impact.", - ) - - return functional_impact +def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/functional-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement") -# TODO#416: For now, this route supports only one evidence line per mapped variant. Eventually, we should support the possibility of multiple evidence lines. @router.get( "/{urn}/va/pathogenicity-statement", - status_code=200, - response_model=VariantPathogenicityStatement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec EvidenceLine from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/pathogenicity-statement", ) -async def show_mapped_variant_acmg_evidence_line( - *, - urn: str, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), - principal: Principal = Depends(get_principal), -) -> VariantPathogenicityStatement: - """ - Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant, principal=principal) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for mapped variant {urn}: {e}") - - if not pathogenicity_statement: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; Variant does not have sufficient evidence to evaluate its pathogenicity.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No pathogenicity statement exists for mapped variant {urn}; Variant does not have sufficient evidence to evaluate its pathogenicity.", - ) - - return pathogenicity_statement +def redirect_mapped_variant_acmg_evidence_line(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/pathogenicity-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement") @router.get( "/vrs/{identifier}", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variants by VRS identifier", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/vrs/{identifier}", ) -async def show_mapped_variants_by_identifier( +def redirect_mapped_variants_by_identifier( *, identifier: Annotated[ str, @@ -236,36 +85,12 @@ async def show_mapped_variants_by_identifier( regex=GA4GH_IR_REGEXP, ), ], - only_current: bool = True, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Fetch a single mapped variant by GA4GH identifier. - """ - query = select(MappedVariant).where( - or_(MappedVariant.pre_mapped["id"].astext == identifier, MappedVariant.post_mapped["id"].astext == identifier) - ) - - if only_current: - query = query.where(MappedVariant.current.is_(True)) + request: Request, +) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/vrs/{identifier}`` instead. - items = db.scalars(query).all() - - permitted_items = [item for item in items if has_permission(user, item.variant.score_set, Action.READ).permitted] - if not permitted_items: - raise HTTPException(status_code=404, detail=f"No mapped variants with identifier {identifier} were found") - - return permitted_items - - -# for testing only -# @router.post("/map/{urn}", status_code=200, responses={404: {}, 500: {}}) -# async def map_score_set(*, urn: str, worker: ArqRedis = Depends(deps.get_worker)) -> Any: -# await worker.lpush(MAPPING_QUEUE_NAME, urn) # type: ignore -# await worker.enqueue_job( -# "variant_mapper_manager", -# None, -# None, -# None -# ) + Note that the replacement's ``only_current`` boolean query parameter has been superseded by + ``as_of``; a caller relying on ``only_current=false`` should switch to passing an explicit + ``as_of`` timestamp rather than expecting it to carry over through this redirect. + """ + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 843db6f1b..4ad6d7228 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -10,7 +10,7 @@ import pandas as pd import requests from arq import ArqRedis -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile +from fastapi import APIRouter, Depends, File, Query, Request, Response, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.exceptions import HTTPException, RequestValidationError from fastapi.responses import StreamingResponse @@ -19,7 +19,7 @@ from pydantic import ValidationError from sqlalchemy import or_, select from sqlalchemy.exc import IntegrityError, MultipleResultsFound -from sqlalchemy.orm import Session, contains_eager +from sqlalchemy.orm import Session from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client @@ -28,6 +28,7 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, @@ -35,6 +36,7 @@ require_current_user, require_current_user_with_email, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.csv.columns import variants_to_csv_rows from mavedb.lib.csv.deprecated_params import ( @@ -47,26 +49,24 @@ from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets +from mavedb.lib.gnomad import get_gnomad_variants_with_variant_urns from mavedb.lib.identifiers import ( create_external_gene_identifier_offset, find_or_create_doi_identifier, find_or_create_publication_identifier, ) from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - correlation_id_for_context, - logging_context, - save_to_logging_context, -) +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.score_set_variants import get_lean_score_set_variants from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, - get_current_mapped_variants_for_annotation, + get_annotatable_variants, is_replaces_id_unique_violation, refresh_variant_urns, ) @@ -82,15 +82,12 @@ generate_experiment_urn, generate_score_set_urn, ) -from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.clinical_control import ClinicalControl +from mavedb.lib.variant_detail import get_variant_detail +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -101,19 +98,21 @@ ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, BASE_409_RESPONSE, + BASE_RESPONSES, GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set +from mavedb.view_models import clinical_control, gnomad_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.doi_identifier import DoiIdentifierCreate +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -122,37 +121,6 @@ SCORE_SET_SEARCH_MAX_PUBLICATION_IDENTIFIERS = 40 -async def enqueue_pipeline_entrypoint( - *, - db: Session, - worker: ArqRedis, - pipeline_name: str, - creating_user: Any, - pipeline_params: dict[str, Any], -) -> None: - pipeline_factory = PipelineFactory(session=db) - pipeline: Optional[Pipeline] = None - - try: - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=creating_user, - pipeline_params=pipeline_params, - ) - job = await worker.enqueue_job( - pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint) - ) - except Exception: - pipeline_factory.discard_pipeline(pipeline) - raise - - if job is None: - logger.info(msg=f"Pipeline entrypoint for {pipeline_name} has already been enqueued.", extra=logging_context()) - else: - save_to_logging_context({"worker_job_id": job.job_id}) - logger.info(msg=f"Enqueued pipeline entrypoint for {pipeline_name}.", extra=logging_context()) - - async def enqueue_variant_creation( *, item: ScoreSet, @@ -220,15 +188,13 @@ async def enqueue_variant_creation( ) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="validate_map_annotate_score_set", - creating_user=user_data.user, - pipeline_params={ - "correlation_id": correlation_id_for_context(), - "score_set_id": item.id, - "updater_id": user_data.user.id, + score_set=item, + user=user_data.user, + extra_params={ "scores_file_key": scores_file_key, "counts_file_key": counts_file_key, "score_columns_metadata": item.dataset_columns.get("score_columns_metadata") @@ -926,9 +892,17 @@ async def show_score_set( def get_score_set_csv_namespaces( *, urn: str, + response: Response, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the offered namespaces as they stood at this instant, so discovery matches an " + "`as_of` download. ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), ) -> Any: """ List the CSV column namespaces this score set has data for, labeled and grouped for a picker. @@ -961,10 +935,190 @@ def get_score_set_csv_namespaces( assert_permission(user_data, score_set, Action.READ) + # Echoed like the CSV endpoints: the instant discovery answered for is part of the answer. + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" return available_score_set_csv_namespaces( db, score_set, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, + ) + + +@router.get( + "/score-sets/{urn}/variants", + status_code=200, + response_model=List[LeanVariant], + response_model_exclude_none=True, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Get the lean whole-set variant view for a score set", +) +async def get_score_set_lean_variants( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (mapping, allele links, VEP consequence) as it stood at " + "this instant, over the score set's fixed scores. ISO 8601, ideally timezone-aware. This is " + "content valid-time only — it never re-selects a score-set version. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return the lean whole-set view for a score set: one pre-chewed record per variant carrying the + selection key (variant URN), score, a representative consequence, the bridge identifiers into the + annotation dimensions (ClinGen allele id, assay-level digest), and the DNA + protein parsed + position/ref/alt blocks that drive the heatmap's level toggle. + + The full set is returned in one payload — the score-set page bins/sorts/filters across every + variant client-side. as_of time-travels the annotation layer only (scores are immutable); the + resolved value is echoed in the X-As-Of response header so the content-time is a visible fact. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "lean-variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + return get_lean_score_set_variants(db, score_set, as_of=as_of) + + +def _stream_score_set_variant_details( + db: Session, + variants: Sequence[Variant], + superseding_score_set: Optional[ScoreSet], + visible_calibration_ids: set[int], + *, + as_of: Optional[datetime] = None, +): + """Serialize the whole-set variant-detail export as NDJSON — one :class:`VariantDetail` per line. + + Assembles each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS + pair + the spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) + one at a time, so a large score set streams rather than building every envelope up front. + ``superseding_score_set`` / ``visible_calibration_ids`` are resolved once for the whole set and + threaded into every per-variant build. + """ + for variant in variants: + detail = get_variant_detail( + db, + variant, + superseding_score_set=superseding_score_set, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, + ) + # get_variant_detail returns the lib transit dataclass; coerce it through the view model. + # exclude_none=False keeps a stable key set per line for programmatic consumers. + yield VariantDetail.model_validate(detail).model_dump_json(by_alias=True, exclude_none=False) + "\n" + + +@router.get( + "/score-sets/{urn}/variant-details", + status_code=200, + responses={ + 200: { + "content": {"application/x-ndjson": {}}, + "description": ( + "Newline-delimited JSON: one VariantDetail per mapped variant — the same envelope the " + "single-variant GET /variants/{urn} route serves, carrying the flat preMapped/postMapped " + "VRS pair, the spec-pure GA4GH CategoricalVariant, and the digest-keyed VEP/gnomAD/ClinVar " + "annotation map." + ), + }, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Download a score set's variant details (VRS + Cat-VRS + annotations)", +) +async def get_score_set_variant_details( + *, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (VRS, Cat-VRS membership + VEP/gnomAD/ClinVar annotations) " + "as it stood at this instant, over the score set's fixed scores. ISO 8601, ideally " + "timezone-aware. Content valid-time only — it never re-selects a score-set version. Defaults " + "to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """Download the score set's variant details — the whole-set streaming pair of the single-variant + ``GET /variants/{urn}`` detail endpoint, and the substrate-faithful replacement for the retired + ``/mapped-variants`` export. + + One record per *mapped* variant (unmapped variants carry no VRS and are omitted): the same + VariantDetail envelope the single-variant route serves — the flat ``preMapped``/``postMapped`` VRS + pair for VRS consumers, plus the spec-pure GA4GH CategoricalVariant and the digest-keyed + VEP/gnomAD/ClinVar annotation map for the full molecular picture. + + Streamed as NDJSON (like the annotated-variant exports) so a large score set downloads without + materializing every envelope server-side and a client can process it line by line. ``as_of`` + time-travels the molecular layer only (scores/classifications are immutable); the resolved value is + echoed in ``X-As-Of`` and the variant count in ``X-Total-Count``. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "variant-details", "as_of": as_of}) + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + + # Resolved here rather than inherited: fetch_score_set_by_urn returns the score set as it actually + # is, leaving narrowing to each response path. Mirrors the single-variant route, which must agree + # with this one -- they serve the same envelope. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + # A calibration's READ rule is stricter than its score set's, so reading the score set does not + # entitle a caller to a private calibration's classifications. + visible_calibration_ids = { + sc.id + for sc in score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted + } + variants = get_annotatable_variants(db, score_set, as_of=as_of) + return StreamingResponse( + _stream_score_set_variant_details( + db, + variants, + superseding, + visible_calibration_ids, + as_of=as_of, + ), + media_type="application/x-ndjson", + headers={ + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), + "X-Processing-Started": datetime.now().isoformat(), + "X-Stream-Type": "variant-detail", + "Access-Control-Expose-Headers": "X-As-Of, X-Total-Count, X-Processing-Started, X-Stream-Type", + }, + ) + + +@router.get( + "/score-sets/{urn}/mapped-variants", + status_code=410, + deprecated=True, + responses={410: BASE_RESPONSES[410]}, + summary="Removed; see GET /score-sets/{urn}/variant-details", +) +def get_score_set_mapped_variants_removed(*, urn: str) -> Any: + """This endpoint has been permanently removed. + + Its JSON-array response has been replaced by a streaming NDJSON payload with a different + field shape (flat ``preMapped``/``postMapped`` VRS pair rather than a ``MappedVariant``-keyed + record), so the two are not wire-compatible and this route does not redirect. Use + ``GET /score-sets/{urn}/variant-details`` instead. + """ + raise HTTPException( + status_code=410, + detail=( + f"GET /score-sets/{urn}/mapped-variants has been removed. " + f"Use GET /score-sets/{urn}/variant-details instead." + ), ) @@ -999,6 +1153,14 @@ def get_score_set_variants_csv( include_custom_columns: Optional[bool] = Query( default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this " + "instant, over the variant's immutable submitted HGVS/scores/counts. ISO 8601, ideally " + "timezone-aware. No effect on the scores/counts namespaces. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1058,6 +1220,8 @@ def get_score_set_variants_csv( "resource_property": "scores", "start": start, "limit": limit, + "drop_na_columns": drop_na_columns, + "as_of": as_of, "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -1080,15 +1244,26 @@ def get_score_set_variants_csv( db, score_set, namespaces, - True, - start, - limit, - drop_unused_hgvs_columns, + namespaced=True, + start=start, + limit=limit, + drop_unused_hgvs_columns_flag=drop_unused_hgvs_columns, + as_of=as_of, # Asked separately from the score set: a private calibration is readable only by its owner, # investigator contributors, or an admin, whoever can read the score set. viewer=principal.viewer_for(ScoreCalibrationViewer), ) - return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) + # Both: the deprecation notice (when a legacy parameter was used) and the resolved content-time, + # so a CSV's as-of instant is a visible fact rather than something the caller has to remember. + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={ + **deprecated.response_headers, + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "Access-Control-Expose-Headers": "X-As-Of", + }, + ) @router.get( @@ -1223,56 +1398,11 @@ async def get_score_set_counts_csv( return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) -@router.get( - "/score-sets/{urn}/mapped-variants", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Get mapped variants from score set by URN", -) -def get_score_set_mapped_variants( - *, - urn: str, - db: Session = Depends(deps.get_db), - user_data: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Return mapped variants from a score set, identified by URN. - """ - save_to_logging_context({"requested_resource": urn, "resource_property": "mapped-variants"}) - - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() - if not score_set: - logger.info( - msg="Could not fetch the requested mapped variants; No such score set exist.", extra=logging_context() - ) - raise HTTPException(status_code=404, detail=f"score set with URN {urn} not found") - - assert_permission(user_data, score_set, Action.READ) - - mapped_variants = ( - db.query(MappedVariant) - .filter(ScoreSet.urn == urn) - .filter(ScoreSet.id == Variant.score_set_id) - .filter(Variant.id == MappedVariant.variant_id) - .all() - ) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variant associated with score set URN {urn} was found", - ) - - return mapped_variants - - def _annotation_stream_record( - mapped_variant, annotation_function + db, variant, annotation_function, as_of=None ) -> tuple[dict, Literal["annotated", "unannotated", "errored"]]: """ - Build the NDJSON record for one mapped variant, and classify its outcome. + Build the NDJSON record for one variant, and classify its outcome. Returns the record together with one of ``annotated``, ``unannotated``, or ``errored``. Nothing raises: a failure past the first record would end the body mid-stream, and since the 200 and its headers went @@ -1281,13 +1411,16 @@ def _annotation_stream_record( failing serialization. A variant with nothing to annotate is *not* an error. It is an expected absence, reported as a null - annotation. Which exceptions mean that is defined once, in ``EXPECTED_ABSENCE_EXCEPTIONS``, because - the corpus sweep has to draw the same line and the two must not drift apart. + annotation: either no annotation context could be built from the mapping substrate at ``as_of``, or + the builder declined the variant. Which exceptions mean that is defined once, in + ``EXPECTED_ABSENCE_EXCEPTIONS``, because the corpus sweep has to draw the same line and the two must + not drift apart. """ - variant_urn = mapped_variant.variant.urn + variant_urn = variant.urn try: - annotation = annotation_function(mapped_variant) + context = variant_annotation_context(db, variant, as_of=as_of) + annotation = annotation_function(context) if context is not None else None annotation_data = annotation.model_dump(exclude_none=True) if annotation else None except EXPECTED_ABSENCE_EXCEPTIONS: logger.debug(f"Nothing to annotate for variant {variant_urn}.") @@ -1309,16 +1442,19 @@ def _annotation_stream_record( return {"variant_urn": variant_urn, "annotation": annotation_data}, "annotated" -def _stream_generated_annotations(mapped_variants, annotation_function): +def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. + Builds each variant's :class:`VariantAnnotationContext` from the mapping-record substrate and passes it + to ``annotation_function``; variants with no live mapping data yield a null annotation. + Metadata should be provided via HTTP headers: - X-Total-Count: Total number of variants - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Type of annotation being streamed - Emits exactly one record per mapped variant, so a body holding fewer lines than ``X-Total-Count`` is + Emits exactly one record per annotatable variant, so a body holding fewer lines than ``X-Total-Count`` is a truncated one. Outcome counts are logged rather than appended to the body, which keeps every line a variant record and keeps this format identical to the public dump's ``va/{urn}.va.ndjson``. @@ -1326,13 +1462,13 @@ def _stream_generated_annotations(mapped_variants, annotation_function): consumed via Server-Sent Events if needed. """ start_time = time.time() - total_variants = len(mapped_variants) + total_variants = len(variants) processed_count = 0 outcome_counts = {"annotated": 0, "unannotated": 0, "errored": 0} - logger.info(f"Starting streaming processing of {total_variants} mapped variants") + logger.info(f"Starting streaming processing of {total_variants} variants") - for mv in mapped_variants: - result, outcome = _annotation_stream_record(mv, annotation_function) + for variant in variants: + result, outcome = _annotation_stream_record(db, variant, annotation_function, as_of=as_of) outcome_counts[outcome] += 1 yield json.dumps(result, default=str) + "\n" @@ -1381,11 +1517,11 @@ def _stream_generated_annotations(mapped_variants, annotation_function): status_code=200, response_model=dict[str, Optional[VariantPathogenicityStatement]], response_model_exclude_none=True, - summary="Get pathogenicity statement annotations for mapped variants within a score set", + summary="Get pathogenicity statement annotations for variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream pathogenicity statement annotations for mapped variants.", + "description": "Stream pathogenicity statement annotations for variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1393,6 +1529,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): def get_score_set_annotated_variants( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1400,16 +1545,16 @@ def get_score_set_annotated_variants( """ Retrieve annotated variants with pathogenicity statements for a given score set. - This endpoint streams pathogenicity evidence lines for all current mapped variants + This endpoint streams pathogenicity evidence lines for all current annotated variants associated with a specific score set. The response is returned as newline-delimited JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to an annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Pathogenicity evidence line details } @@ -1421,7 +1566,7 @@ def get_score_set_annotated_variants( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1439,23 +1584,27 @@ def get_score_set_annotated_variants( Returns: Any: StreamingResponse containing newline-delimited JSON with pathogenicity - evidence lines for each mapped variant. Response includes headers with + evidence lines for each annotated variant. Response includes headers with total count, processing start time, and stream type information. + A score set that exists but has no annotatable variants (never mapped, or none live at ``as_of``) + streams an empty body with ``X-Total-Count: 0`` — an empty collection, not a 404. + Raises: HTTPException: 404 error if the score set with the given URN is not found. - HTTPException: 404 error if no mapped variants are associated with the score set. HTTPException: 403 error if the user lacks READ permissions for the score set. Note: This function logs the request context and validates user permissions before - processing. Only current (non-historical) mapped variants are included in - the response. + processing. Use the `as_of` parameter to reconstruct the molecular layer as it stood at a specific + instant, over the variant's fixed score. The response is streamed to allow for efficient handling + of large datasets, and progress updates are logged for monitoring purposes. """ save_to_logging_context( { "requested_resource": urn, "resource_property": "annotated-variants/pathogenicity-statement", + "as_of": as_of, } ) @@ -1469,20 +1618,19 @@ def get_score_set_annotated_variants( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct evidence lines.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, partial(variant_pathogenicity_statement, principal=principal)), + _stream_generated_annotations( + db, variants, partial(variant_pathogenicity_statement, principal=principal), as_of=as_of + ), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "pathogenicity-evidence-line", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1495,11 +1643,11 @@ def get_score_set_annotated_variants( status_code=200, response_model=dict[str, Optional[Statement]], response_model_exclude_none=True, - summary="Get functional impact statement annotations for mapped variants within a score set", + summary="Get functional impact statement annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional impact statement annotations for mapped variants.", + "description": "Stream functional impact statement annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1507,6 +1655,15 @@ def get_score_set_annotated_variants( def get_score_set_annotated_variants_functional_statement( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1514,16 +1671,16 @@ def get_score_set_annotated_variants_functional_statement( """ Retrieve functional impact statements for annotated variants in a score set. - This endpoint streams functional impact statements for all current mapped variants + This endpoint streams functional impact statements for all current annotated variants associated with a specific score set. The response is delivered as newline-delimited JSON (NDJSON) format. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to an annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional impact statement details } @@ -1535,7 +1692,7 @@ def get_score_set_annotated_variants_functional_statement( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1551,20 +1708,28 @@ def get_score_set_annotated_variants_functional_statement( Returns: StreamingResponse: NDJSON stream containing functional impact statements for each - mapped variant. Response includes headers with total count, processing start time, + annotated variant. Response includes headers with total count, processing start time, and stream type information. Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Note: - Only current (non-historical) mapped variants are included in the response. - The function requires appropriate read permissions on the score set. + The function requires appropriate read permissions on the score set. Use the `as_of` + parameter to reconstruct the molecular layer as it stood at a specific instant, over + the variant's fixed score. The response is streamed to allow for efficient handling of + large datasets, and progress updates are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/functional-statement"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/functional-statement", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1576,22 +1741,19 @@ def get_score_set_annotated_variants_functional_statement( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct functional impact statements.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( _stream_generated_annotations( - mapped_variants, partial(variant_functional_impact_statement, principal=principal) + db, variants, partial(variant_functional_impact_statement, principal=principal), as_of=as_of ), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-impact-statement", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1604,11 +1766,11 @@ def get_score_set_annotated_variants_functional_statement( status_code=200, response_model=dict[str, Optional[ExperimentalVariantFunctionalImpactStudyResult]], response_model_exclude_none=True, - summary="Get functional study result annotations for mapped variants within a score set", + summary="Get functional study result annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional study result annotations for mapped variants.", + "description": "Stream functional study result annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1616,22 +1778,31 @@ def get_score_set_annotated_variants_functional_statement( def get_score_set_annotated_variants_functional_study_result( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional study results for annotated variants in a score set. - This endpoint streams functional study result annotations for all current mapped variants + This endpoint streams functional study result annotations for all current annotated variants associated with a specific score set. The results are returned as newline-delimited JSON (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to a annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional study result details } @@ -1643,7 +1814,7 @@ def get_score_set_annotated_variants_functional_study_result( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1660,7 +1831,7 @@ def get_score_set_annotated_variants_functional_study_result( Returns: StreamingResponse: A streaming response containing functional study results in NDJSON format. Headers include: - - X-Total-Count: Total number of mapped variants being streamed + - X-Total-Count: Total number of annotated variants being streamed - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Set to "functional-study-result" - Access-Control-Expose-Headers: Exposed headers for CORS @@ -1668,15 +1839,22 @@ def get_score_set_annotated_variants_functional_study_result( Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Notes: - - Only returns current mapped variants (MappedVariant.current == True) - - Eagerly loads related ScoreSet data including publications, users, license, and experiment - - Logs requests and errors for monitoring and debugging purposes + - The `as_of` parameter allows reconstruction of the molecular layer as it stood at a specific + instant, over the variant's fixed score. It is ISO 8601 formatted and ideally timezone-aware. + - The response is streamed to allow for efficient handling of large datasets, and progress updates + are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/study-result"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/study-result", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1688,20 +1866,17 @@ def get_score_set_annotated_variants_functional_study_result( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct study results.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_study_result), + _stream_generated_annotations(db, variants, variant_study_result, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-study-result", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -2589,12 +2764,12 @@ async def publish_score_set( db.refresh(item) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="publish_score_set", - creating_user=user_data.user, - pipeline_params={"correlation_id": correlation_id_for_context(), "score_set_id": item.id}, + score_set=item, + user=user_data.user, ) except Exception as exc: logger.warning( @@ -2609,7 +2784,7 @@ async def publish_score_set( @router.get( "/score-sets/{urn}/clinical-controls", status_code=200, - response_model=list[clinical_control.ClinicalControlWithMappedVariants], + response_model=list[clinical_control.ClinicalControlWithClinvarLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get clinical controls for a score set", @@ -2617,20 +2792,33 @@ async def publish_score_set( async def get_clinical_controls_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. _db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinicalControl]: +) -> list[clinical_control.ClinicalControlWithClinvarLinks]: """ Fetch relevant clinical controls for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version + if db_name is not None: + save_to_logging_context({"db_name": db_name}) + if db_version is not None: + save_to_logging_context({"db_version": db_version}) item: Optional[ScoreSet] = _db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2642,26 +2830,13 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinicalControl) - .join(ClinicalControl.mapped_variants) - .join(MappedVariant.variant) - .options(contains_eager(ClinicalControl.mapped_variants).contains_eager(MappedVariant.variant)) - .filter(MappedVariant.current.is_(True)) - .filter(Variant.score_set_id == item.id) + controls = get_clinical_controls_with_variant_urns( + _db, item.id, as_of=as_of, db_name=db_name, db_version=db_version ) - if db_name is not None: - save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_name == db_name) - - if db_version is not None: - save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_version == db_version) - - clinical_controls: Sequence[ClinicalControl] = _db.scalars(clinical_controls_query).unique().all() - - if not clinical_controls: + if not controls: + # Can legitimately fire even for a (db_name, db_version) sourced from `.../options`: liveness + # is re-evaluated per call, see that endpoint's docstring. logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2671,9 +2846,27 @@ async def get_clinical_controls_for_score_set( detail=f"No clinical control variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(clinical_controls)}) + save_to_logging_context({"resource_count": len(controls)}) - return clinical_controls + return [ + clinical_control.ClinicalControlWithClinvarLinks.model_validate( + { + "id": ctrl.id, + "db_identifier": ctrl.db_identifier, + "gene_symbol": ctrl.gene_symbol, + "clinical_significance": ctrl.clinical_significance, + "clinical_review_status": ctrl.clinical_review_status, + "db_version": ctrl.db_version, + "db_name": ctrl.db_name, + "modification_date": ctrl.modification_date, + "creation_date": ctrl.creation_date, + "clinvar_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for ctrl, links in controls + ] @router.get( @@ -2687,14 +2880,33 @@ async def get_clinical_controls_for_score_set( async def get_clinical_controls_options_for_score_set( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), ) -> list[dict[str, Union[str, list[str]]]]: """ Fetch clinical control options for a given score set. + + Each ``(db_name, db_version)`` pair returned here was live at the moment of this call, but + liveness is re-evaluated independently per request. A pair fetched here can have its backing + ``ClinvarAlleleLink`` retired before a later call to ``GET /score-sets/{urn}/clinical-controls`` + filters on it, in which case that call 404s. Pin an explicit ``as_of`` on both calls to avoid this + possibility. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_control_options"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "clinical_control_options", + "as_of": as_of, + } + ) item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2706,23 +2918,9 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinicalControl.db_name, ClinicalControl.db_version) - .join(MappedVariant, ClinicalControl.mapped_variants) - .join(Variant) - .where(MappedVariant.current.is_(True)) - .where(Variant.score_set_id == item.id) - ) - - clinical_controls_for_item = db.execute(clinical_controls_query).unique() - - # NOTE: We return options even for pairwise groupings which may have no associated mapped variants - # and 404 when ultimately requested together. - clinical_control_options: dict[str, list[str]] = {} - for db_name, db_version in clinical_controls_for_item: - clinical_control_options.setdefault(db_name, []).append(db_version) + options = get_clinical_control_options(db, item.id, as_of=as_of) - if not clinical_control_options: + if not options: logger.info( msg="Failed to fetch clinical control options for score set; No clinical control variants are associated with this score set.", extra=logging_context(), @@ -2732,16 +2930,13 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) - return [ - dict(zip(("db_name", "available_versions"), (db_name, db_versions))) - for db_name, db_versions in clinical_control_options.items() - ] + return [{"db_name": db_name, "available_versions": available_versions} for db_name, available_versions in options] @router.get( "/score-sets/{urn}/gnomad-variants", status_code=200, - response_model=list[gnomad_variant.GnomADVariantWithMappedVariants], + response_model=list[gnomad_variant.GnomADVariantWithVariantLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get gnomad variants for a score set", @@ -2749,16 +2944,26 @@ async def get_clinical_controls_options_for_score_set( async def get_gnomad_variants_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → gnomAD link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), version: Optional[str] = None, -) -> Sequence[GnomADVariant]: +) -> list[gnomad_variant.GnomADVariantWithVariantLinks]: """ - Fetch relevant gnomad variants for a given score set. + Fetch relevant gnomad variants for a given score set, each paired with the score-set variants (and + annotated allele digests) it links to over the allele substrate. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_version = version item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() @@ -2771,28 +2976,12 @@ async def get_gnomad_variants_for_score_set( assert_permission(user_data, item, Action.READ) - gnomad_variants_query = ( - select(GnomADVariant) - .join(MappedVariant, GnomADVariant.mapped_variants) - .join(Variant) - .where(Variant.score_set_id == item.id) - ) - if db_version is not None: save_to_logging_context({"db_version": db_version}) - gnomad_variants_query = gnomad_variants_query.where(GnomADVariant.db_version == db_version) - gnomad_variants_for_item: Sequence[GnomADVariant] = db.scalars(gnomad_variants_query).all() - gnomad_variants_with_mapped_variant = [] - for gnomad_variant_in_item in gnomad_variants_for_item: - gnomad_variant_in_item.mapped_variants = [ - mv for mv in gnomad_variant_in_item.mapped_variants if mv.current and mv.variant.score_set_id == item.id - ] - - if gnomad_variant_in_item.mapped_variants: - gnomad_variants_with_mapped_variant.append(gnomad_variant_in_item) + gnomad_variants = get_gnomad_variants_with_variant_urns(db, item.id, as_of=as_of, db_version=db_version) - if not gnomad_variants_with_mapped_variant: + if not gnomad_variants: logger.info( msg="No gnomad variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2802,6 +2991,26 @@ async def get_gnomad_variants_for_score_set( detail=f"No gnomad variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(gnomad_variants_for_item)}) + save_to_logging_context({"resource_count": len(gnomad_variants)}) - return gnomad_variants_for_item + return [ + gnomad_variant.GnomADVariantWithVariantLinks.model_validate( + { + "id": gv.id, + "db_name": gv.db_name, + "db_identifier": gv.db_identifier, + "db_version": gv.db_version, + "allele_count": gv.allele_count, + "allele_number": gv.allele_number, + "allele_frequency": gv.allele_frequency, + "faf95_max": gv.faf95_max, + "faf95_max_ancestry": gv.faf95_max_ancestry, + "creation_date": gv.creation_date, + "modification_date": gv.modification_date, + "variant_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for gv, links in gnomad_variants + ] diff --git a/src/mavedb/routers/shared.py b/src/mavedb/routers/shared.py index f98edb39c..2cddfd23d 100644 --- a/src/mavedb/routers/shared.py +++ b/src/mavedb/routers/shared.py @@ -8,6 +8,7 @@ 403: {"description": "Forbidden. Insufficient permissions."}, 404: {"description": "Resource not found."}, 409: {"description": "Conflict with current resource state."}, + 410: {"description": "Gone. This resource has been permanently removed."}, 416: {"description": "Requested range not satisfiable."}, 422: {"description": "Unprocessable entity. Validation failed."}, 429: {"description": "Too many requests. Rate limit exceeded."}, @@ -23,6 +24,7 @@ BASE_403_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {403: BASE_RESPONSES[403]} BASE_404_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {404: BASE_RESPONSES[404]} BASE_409_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {409: BASE_RESPONSES[409]} +BASE_410_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {410: BASE_RESPONSES[410]} BASE_416_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {416: BASE_RESPONSES[416]} BASE_422_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {422: BASE_RESPONSES[422]} BASE_429_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {429: BASE_RESPONSES[429]} diff --git a/src/mavedb/routers/statistics.py b/src/mavedb/routers/statistics.py index e52d2582d..5deea5636 100644 --- a/src/mavedb/routers/statistics.py +++ b/src/mavedb/routers/statistics.py @@ -345,7 +345,7 @@ def record_mapped_variant_counts(db: Session = Depends(get_db)) -> dict[str, int within a given record. """ variants = db.execute( - select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapped_variant_id)) + select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapping_record_id)) .group_by(PublishedVariantsMV.score_set_urn) .order_by(PublishedVariantsMV.score_set_urn) ).all() @@ -650,10 +650,10 @@ def mapped_variant_counts( """ # Fast path: total distinct mapped variants (optionally only current) without per-date aggregation. if group is None: - total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapped_variant_id))) + total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapping_record_id))) if onlyCurrent: - total_stmt = total_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + total_stmt = total_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) total = db.execute(total_stmt).scalar_one() # type: ignore return OrderedDict([("count", total)]) @@ -661,11 +661,11 @@ def mapped_variant_counts( # Grouped path: materialize distinct counts per published_date, then roll up. per_date_stmt = select( PublishedVariantsMV.published_date, - func.count(func.distinct(PublishedVariantsMV.mapped_variant_id)), + func.count(func.distinct(PublishedVariantsMV.mapping_record_id)), ) if onlyCurrent: - per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) per_date = db.execute( per_date_stmt.group_by(PublishedVariantsMV.published_date).order_by(PublishedVariantsMV.published_date) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index bb76716c4..fe0ce2001 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,31 +1,38 @@ -import itertools import logging -import re -from typing import Any, List, Optional +from datetime import datetime +from typing import Annotated, Any, List, Optional -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Path, Query, Response from fastapi.exceptions import HTTPException from fastapi.responses import StreamingResponse +from ga4gh.core.identifiers import GA4GH_IR_REGEXP +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session, joinedload -from sqlalchemy.sql import or_ +from sqlalchemy.orm import Session from mavedb import deps +from mavedb.lib.alleles import find_variants_by_vrs_identifier +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authentication import get_current_user from mavedb.lib.authorization import get_principal from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData -from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet +from mavedb.lib.variant_detail import get_variant_detail from mavedb.models.variant import Variant -from mavedb.models.variant_translation import VariantTranslation from mavedb.routers.shared import ( ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, @@ -33,11 +40,8 @@ ROUTER_BASE_PREFIX, ) from mavedb.view_models.csv_namespace import AvailableCsvNamespace -from mavedb.view_models.variant import ( - ClingenAlleleIdVariantLookupResponse, - ClingenAlleleIdVariantLookupsRequest, - VariantEffectMeasurementWithScoreSet, -) +from mavedb.view_models.variant import VariantVrsMatch +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -56,418 +60,305 @@ } -@router.post( - "/variants/clingen-allele-id-lookups", +def _fetch_readable_variant(db: Session, user_data: Optional[UserData], urn: str) -> Variant: + """Fetch a single variant by URN and assert the caller may read its score set. + + Raises 404 when no such variant exists, 500 when the URN is not unique (an invariant break), and the + standard 403 (via ``assert_permission``) when the score set is not readable. + """ + try: + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + return variant + + +@router.get( + "/variants/vrs/{identifier}", status_code=200, - response_model=list[ClingenAlleleIdVariantLookupResponse], - responses={ - **BASE_400_RESPONSE, - **ACCESS_CONTROL_ERROR_RESPONSES, - }, - summary="Lookup variants by ClinGen Allele IDs", + response_model=list[VariantVrsMatch], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Look up variants by VRS identifier", ) -def lookup_variants( +def lookup_variants_by_vrs_identifier( *, - request: ClingenAlleleIdVariantLookupsRequest, + response: Response, + identifier: Annotated[ + str, + Path( + description="A valid GA4GH digest-based identifier for the mapped allele.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), - user_data: UserData = Depends(get_current_user), -): + user_data: Optional[UserData] = Depends(get_current_user), +) -> list[VariantVrsMatch]: + """Resolve a GA4GH VRS identifier to the readable variants whose mapping links that allele. + + A deduplicated allele may be shared across score sets, so one identifier can resolve to several + variants. This is a lookup returning a collection: results are filtered to the score sets the caller + may read, and an empty list is returned when nothing readable matches. An absent identifier and a + match visible only in a private score set are deliberately indistinguishable (both yield ``[]``), so + the response never reveals a private allele's existence. """ - Lookup variants by ClinGen Allele IDs. + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + matches = find_variants_by_vrs_identifier(db, identifier, as_of=as_of) + permitted = [ + (variant, allele) + for variant, allele in matches + if has_permission(user_data, variant.score_set, Action.READ).permitted + ] + + return [ + VariantVrsMatch( + variant_urn=variant.urn or "", + clingen_allele_id=allele.clingen_allele_id, + vrs_id=(allele.post_mapped or {}).get("id"), + level=allele.level, + ) + for variant, allele in permitted + ] + + +@router.get( + "/variants/{urn}", + status_code=200, + response_model=VariantDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + # Emit nulls rather than omitting absent fields: this shape is a stable, self-describing envelope + # (its bulk pair, GET /score-sets/{urn}/variant-details, streams the same object one-per-line for + # dataframe/columnar consumption), so every response carries the same key set. + response_model_exclude_none=False, + summary="Fetch assayed variant detail by URN", +) +def get_variant( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """Fetch the two-tier detail envelope for a single assayed variant by URN. + + Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a + digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is + the citable unit) but self-describes via isCurrent/supersededByScoreSet rather than reading as current. """ - save_to_logging_context({"requested_resource": "clingen-allele-id-lookups"}) - save_to_logging_context({"clingen_allele_ids_to_lookup": request.clingen_allele_ids}) - logger.debug(msg="Looking up variants by Clingen Allele IDs", extra=logging_context()) - - # sort multi-variant components lexicographically, as they are in the database - request.clingen_allele_ids = [",".join(sorted(allele_id.split(","))) for allele_id in request.clingen_allele_ids] - exact_match_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(request.clingen_allele_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - save_to_logging_context({"num_variants_matching_clingen_allele_ids": len(exact_match_variants)}) - logger.debug(msg="Found variants with exactly matching ClinGen Allele IDs", extra=logging_context()) - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - - variants_by_allele_id: dict[str, dict] = { - allele_id: { - "clingen_allele_id": allele_id, - "exact_match": {"clingen_allele_id": allele_id, "variant_effect_measurements": []}, - "equivalent_nt": [], - "equivalent_aa": [], - } - for allele_id in request.clingen_allele_ids + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + variant = _fetch_readable_variant(db, user_data, urn) + + # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn + # does — a newer version the user cannot read is not leaked (the variant then reads as current). + superseding = variant.score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + visible_calibration_ids = { + sc.id + for sc in variant.score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted } - for variant, allele_id in exact_match_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - num_variants_matching_clingen_allele_ids_and_permitted += 1 - variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"].append(variant) - save_to_logging_context( - {"clingen_allele_ids_with_permitted_variants": num_variants_matching_clingen_allele_ids_and_permitted} + return get_variant_detail( + db, + variant, + superseding_score_set=superseding, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, ) - for allele_id in request.clingen_allele_ids: - if not variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"]: - variants_by_allele_id[allele_id]["exact_match"] = None - - for allele_id in request.clingen_allele_ids: - # validate and determine whether multi-variant - # NOTE: assuming we never have more than 2 components in a multi-variant - single_clingen_allele_id_re = r"^[CP]A\d+$" - multi_clingen_allele_id_re = r"^[CP]A\d+,[CP]A\d+$" - single_or_multi_clingen_allele_id_re = r"^[CP]A\d+(,[CP]A\d+)?$" - - if re.fullmatch(single_or_multi_clingen_allele_id_re, allele_id) is None: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}'", - ) - - if re.fullmatch(single_clingen_allele_id_re, allele_id): - if allele_id.startswith("PA"): - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where(VariantTranslation.aa_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == allele_id, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_aa_clingen_ids = [ - var_translation.aa_clingen_id - for var_translation in related_clingen_ids - if var_translation.aa_clingen_id != allele_id - ] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_clingen_ids = [var_translation.nt_clingen_id for var_translation in related_clingen_ids] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - elif allele_id.startswith("CA"): - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where(VariantTranslation.nt_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - related_aa_clingen_ids = [var_translation.aa_clingen_id for var_translation in related_clingen_ids] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_nt_clingen_ids = [ - var_translation.nt_clingen_id - for var_translation in related_clingen_ids - if var_translation.nt_clingen_id != allele_id - ] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - elif re.fullmatch(multi_clingen_allele_id_re, allele_id): - # validate each component allele id - already done via re - # allow more than 2 components? - # full matches are determined the same exact way as single variant (done above already) - # equivalent aa/nt: if all components of variant are represented by equivalent or exact match, but not all exact matches because those are already represented above - # so basically go through variant translations table and create every possible combo of components except for full exact match, - # then search the db for those combos - - allele_id_components = allele_id.split(",") - if all(component.startswith("PA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where( - VariantTranslation.aa_clingen_id == component - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == component, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[0]) - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[1]) - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - elif all(component.startswith("CA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where( - VariantTranslation.nt_clingen_id == allele_id - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[0]) - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[1]) - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - else: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}': all components must be either PA or CA", - ) - - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - return [variants_by_allele_id[allele_id] for allele_id in request.clingen_allele_ids] + +@router.get( + "/variants/{urn}/va/study-result", + status_code=200, + response_model=ExperimentalVariantFunctionalImpactStudyResult, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec StudyResult for a variant", +) +def get_variant_study_result( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> ExperimentalVariantFunctionalImpactStudyResult: + """Construct a single VA-Spec StudyResult for a variant by URN, from its mapping substrate.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, detail=f"No study result exists for variant {urn}: no mapping data exists." + ) + + try: + return variant_study_result(context) + except MappingDataDoesntExistException as e: + logger.info(msg=f"Could not construct a study result for variant {urn}: {e}", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"No study result exists for variant {urn}: {e}") @router.get( - "/variants/{urn}", + "/variants/{urn}/va/functional-statement", status_code=200, - response_model=VariantEffectMeasurementWithScoreSet, + response_model=Statement, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - response_model_exclude_none=True, - summary="Fetch variant by URN", + summary="Construct a VA-Spec functional-impact Statement for a variant", ) -def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user)): - """ - Fetch a single variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) +def get_variant_functional_impact_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Statement: + """Construct a single VA-Spec functional-impact Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No functional impact statement exists for variant {urn}: no mapping data exists.", + ) + try: - query = db.query(Variant).filter(Variant.urn == urn) - variant = query.one_or_none() - except MultipleResultsFound: - logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) - raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + functional_impact = variant_functional_impact_statement(context, principal=principal) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a functional impact statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No functional impact statement exists for variant {urn}: {e}") + + if not functional_impact: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its functional impact.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient " + "evidence to evaluate its functional impact." + ), + ) - if not variant: - logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + return functional_impact - assert_permission(user_data, variant.score_set, Action.READ) - return variant + +@router.get( + "/variants/{urn}/va/pathogenicity-statement", + status_code=200, + response_model=VariantPathogenicityStatement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec pathogenicity Statement for a variant", +) +def get_variant_pathogenicity_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> VariantPathogenicityStatement: + """Construct a single VA-Spec pathogenicity Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No pathogenicity statement exists for variant {urn}: no mapping data exists.", + ) + + try: + pathogenicity_statement = variant_pathogenicity_statement(context, principal=principal) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a pathogenicity statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for variant {urn}: {e}") + + if not pathogenicity_statement: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its pathogenicity.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient " + "evidence to evaluate its pathogenicity." + ), + ) + + return pathogenicity_statement @router.get( @@ -480,9 +371,17 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User def get_variant_csv_namespaces( *, urn: str, + response: Response, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the offered namespaces as they stood at this instant, so discovery matches an " + "`as_of` download. ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), ) -> Any: """ List the CSV column namespaces this variant has data for, labeled and grouped for a picker. @@ -513,11 +412,14 @@ def get_variant_csv_namespaces( assert_permission(user_data, variant.score_set, Action.READ) + # Echoed like the CSV endpoints: the instant discovery answered for is part of the answer. + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" return available_variant_csv_namespaces( db, urn, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, ) @@ -545,6 +447,14 @@ def get_variant_csv_data( db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the mapping-derived columns (reference HGVS, VEP, gnomAD, ClinVar) as they " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Defaults to current." + ), + ), ) -> Any: """ Return tabular data for a single variant, identified by URN, in CSV format. @@ -598,9 +508,14 @@ def get_variant_csv_data( namespaces=namespaces, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, ) return StreamingResponse( iter([csv_str]), media_type="text/csv", - headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + headers={ + "Content-Disposition": f'attachment; filename="{urn}.csv"', + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "Access-Control-Expose-Headers": "X-As-Of", + }, ) diff --git a/src/mavedb/scripts/calibrated_variant_effects.py b/src/mavedb/scripts/calibrated_variant_effects.py index a8f470886..d209b436d 100644 --- a/src/mavedb/scripts/calibrated_variant_effects.py +++ b/src/mavedb/scripts/calibrated_variant_effects.py @@ -74,7 +74,16 @@ @click.command() @with_database_session def main(db: Session) -> None: - """Count unique variant effect measurements with ACMG-classified functional ranges.""" + """Count unique variant effect measurements with ACMG-classified functional ranges. + + Counts every calibration on a public score set, applying no ``ScoreCalibrationViewer``. A calibration + keeps its own ``private`` flag and a READ rule stricter than its score set's, so some of what is + counted here is not readable by any caller the API would serve. That is deliberate: this is an + operator's census of what MaveDB holds, not a response to a request, and a viewer-filtered count would + under-report the corpus. It is also the aggregate blind spot ``lib/permissions/viewer.py`` names — a + count leaks no calibration's contents, but it does reveal that private ones exist. Keep the output + internal, and do not lift these totals into anything caller-facing without applying a viewer. + """ query = ( select(ScoreSet) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 8d93e3bc3..aa68c5eb8 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -30,14 +30,19 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.csv.entries import score_sets_have_current_mappings from mavedb.lib.csv.namespaces import CsvNamespace from mavedb.lib.csv.score_set import ( available_score_set_csv_namespaces, get_score_set_variants_as_csv, ) +from mavedb.lib.permissions import Action, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.score_sets import get_annotatable_variants from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -64,30 +69,16 @@ def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: """The namespaces the public annotations CSV should carry for this score set. - *viewer* has no default on purpose. Discovery resolves an omitted viewer to the public subset, which - is the right answer for the dump but the wrong way to arrive at it: the archive's audience is a - decision this script makes, so it says so rather than inheriting it. + *viewer* has no default: the dump always targets the anonymous/public audience, and that should be + explicit here rather than inherited from discovery's default. - Asks discovery what the score set actually has rather than naming groups by hand. The previous - hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases - never ingested, needed a code change for every new release, and was fragile to schema changes. + Derived from discovery rather than a hand-maintained list, so a new release needs no code change here. + Includes everything discovery finds regardless of `selected_by_default` (a UI attention flag, not a + completeness one) or `research_use_only` (each group carries its own `research_use_only` column, so + consumers can filter on the data itself — VA-Spec NDJSON follows a different rule, see TODO(#803)). - The archive carries everything MaveDB holds about the score set, so this takes what discovery found - and subtracts from it rather than opting groups in. - - In particular it does not filter on `selected_by_default`. That flag answers "what should a download - dialog open on", which is a question about attention rather than about what exists, and the reasons a - group opens unchecked are not interchangeable. An archive is about completeness, not about what a user - should be nudged to look at first. - - Nor does it filter on `research_use_only`. A research-use-only calibration is public data, and every - group it produces carries a `research_use_only` column stating its standing, so a consumer can filter - on the data itself. VA-Spec NDJSON follows a different rule currently, see TODO(#803). - - Subtractions: - - - Every score and count group, and the score set's own identity: scores and counts get their own - files, and the URN is in the filename, so repeating either would be noise. + Excludes the score, count, and score-set-identity groups: those get their own files, and the URN is + already in the filename. """ excluded = { CsvNamespace.SCORES, @@ -107,20 +98,31 @@ def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: def archive_path_base(score_set_urn: str) -> str: - """The filename stem a score set's artifacts share, e.g. ``urn-mavedb-00000001-a-1``. + """The filename stem shared by a score set's artifacts, e.g. ``urn-mavedb-00000001-a-1``. - Colons are not portable in archive member names on every platform, so the URN is hyphenated. The - README documents the substitution as the way back to the URN, which makes it part of the published - contract rather than an implementation detail. + Colons aren't portable in archive member names on every platform, so the URN is hyphenated. The + README documents the substitution as the way back to the URN. """ return score_set_urn.replace(":", "-") def score_set_has_current_mappings(db: Session, score_set: ScoreSet) -> bool: - """Whether any variant in the score set has a current mapping. + """Whether any variant in the score set has a live mapping on the allele substrate. - Gates the three mapping-derived artifacts. A score set whose mappings are all superseded yields no - annotations, so emitting empty files for it would advertise absence as data. + Gates the artifacts built from that substrate, so a score set whose mappings are all superseded + doesn't emit empty files. Reads the same predicate CSV discovery uses rather than the frozen + ``MappedVariant`` table, which no longer gets writes and would report every score set mapped since + the migration as unmapped. + """ + # TODO(#372): non-null id fields + return score_sets_have_current_mappings(db, [score_set.id]) # type: ignore + + +def score_set_has_legacy_mapped_variants(db: Session, score_set: ScoreSet) -> bool: + """Whether the frozen ``MappedVariant`` table still holds current rows for this score set. + + Gates ``mapped/{urn}.mapped-variants.json``, the one artifact still sourced from that table. Score + sets mapped after the allele-substrate migration have no such rows. """ return ( db.scalars( @@ -152,9 +154,8 @@ def counts_csv(db: Session, score_set: ScoreSet) -> Optional[str]: def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> str: """`csv/{urn}.annotations.csv` — every annotation namespace discovery offers for the score set. - The same *viewer* selects the namespaces and resolves their cells. Threading one viewer through both - is what keeps a calibration from being offered as a column group and then withheld as data, or the - reverse. + The same *viewer* selects the namespaces and resolves their cells, so a calibration can't be offered + as a column group and then withheld as data, or vice versa. """ return get_score_set_variants_as_csv( db, @@ -168,8 +169,11 @@ def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationVi def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: """`mapped/{urn}.mapped-variants.json` — the score set's current mapped variants. - Same shape as GET /api/v1/score-sets/{urn}/mapped-variants, but narrower: that endpoint also - returns superseded mappings, while this dump includes only each variant's current mapping. + Legacy, and the last artifact sourced from the frozen ``MappedVariant`` table. The endpoint this + once mirrored is gone — GET /score-sets/{urn}/mapped-variants now returns 410, and its replacement + serves a different field shape (see ``get_score_set_mapped_variants_removed``). + ``vrs/{urn}.vrs.ndjson`` supersedes this file in the archive. Only each variant's current mapping + is included, never a superseded one. """ mapped_variants = db.scalars( select(MappedVariant) @@ -184,16 +188,20 @@ def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: - """`va/{urn}.va.ndjson` — one record per current mapped variant at its highest materialized VA level. + """`va/{urn}.va.ndjson` — one record per annotatable variant at its highest materialized VA level. - Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams. Every record is - newline-terminated, the last one included, so a line-based consumer needs no special case. + Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams, which select the same set. + Every record is newline-terminated, including the last, so a line-based consumer needs no special + case. A variant the pipeline could not place carries no post-mapped allele, is therefore not + annotatable, and contributes no line; ``annotation`` is null where a placed variant yields no + VA-Spec layer. """ lines = [] - for mv in get_current_mapped_variants_for_annotation(db, score_set): - annotation = variant_highest_level_annotation(mv, principal=principal) + for variant in get_annotatable_variants(db, score_set): + context = variant_annotation_context(db, variant) + annotation = variant_highest_level_annotation(context, principal=principal) if context is not None else None record = { - "variant_urn": mv.variant.urn, + "variant_urn": variant.urn, "annotation": annotation.model_dump(exclude_none=True) if annotation else None, } lines.append(json.dumps(record, default=str)) @@ -201,16 +209,53 @@ def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: return "".join(line + "\n" for line in lines) +def vrs_ndjson(db: Session, score_set: ScoreSet) -> str: + """`vrs/{urn}.vrs.ndjson` — GA4GH VRS objects for each mapped variant, one line each. + + Carries the VRS pair plus the Cat-VRS categorical variant, built from the same live allele links + `GET /variants/{urn}` serves. Its own artifact rather than CSV columns because these are nested + objects; join to `annotations.csv` on `mavedb.post_mapped_vrs_id`. + + `pre_mapped` is the assayed-level VRS on the target's own reference; `post_mapped` is the measured + allele lifted to a genomic or transcript reference. `categorical_variant` is spec-pure Cat-VRS (no + MaveDB fields), anchored on the measured allele with the derived alleles as members — null when + there's no hydratable authoritative allele. + + Emitted for every mapped variant regardless of calibration visibility, since none of this is + calibration-derived. + """ + lines = [] + for variant in get_annotatable_variants(db, score_set): + links = get_live_record_allele_links(db, variant.id) + authoritative = next((link.allele for link in links if link.is_authoritative), None) + record = next((link.mapping_record for link in links), None) + transit = build_categorical_variant(links, name=variant.urn or "") + + lines.append( + json.dumps( + { + "variant_urn": variant.urn, + "pre_mapped": record.pre_mapped if record is not None else None, + "post_mapped": authoritative.post_mapped if authoritative is not None else None, + "categorical_variant": ( + transit.categorical_variant.model_dump(mode="json", exclude_none=True) + if transit is not None + else None + ), + }, + default=str, + ) + ) + + return "".join(line + "\n" for line in lines) + + def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) -> Iterator[tuple[str, str]]: """Every archive entry one score set contributes, as ``(path within the zip, content)`` pairs. - Scores are unconditional. Counts appear only where count columns are defined, and the three - mapping-derived artifacts only where a current mapping exists — see the README's caveats, which - promise exactly this and are what a consumer checks a missing file against. - - A generator rather than a dict so the caller writes each artifact and lets it go. Returning them - together would hold all four of a score set's payloads in memory at once, and one score set's - ``va.ndjson`` alone runs to tens of kilobytes per variant once a pathogenicity layer materializes. + Scores are unconditional; counts and the mapping-derived artifacts appear only when the score set + has them — see the README's caveats. A generator rather than a dict so the caller writes and + releases each artifact, instead of holding a score set's full payload in memory at once. """ base = archive_path_base(str(score_set.urn)) viewer = principal.viewer_for(ScoreCalibrationViewer) @@ -219,35 +264,44 @@ def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) if score_set_has_current_mappings(db, score_set): yield f"csv/{base}.annotations.csv", annotations_csv(db, score_set, viewer) - yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + yield f"vrs/{base}.vrs.ndjson", vrs_ndjson(db, score_set) yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) + if score_set_has_legacy_mapped_variants(db, score_set): + yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + counts = counts_csv(db, score_set) if counts is not None: yield f"csv/{base}.counts.csv", counts def public_experiment_set( - experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] + experiment_set_view: ExperimentSetPublicDump, + visible_calibration_ids: set[int], + readable_superseding_urns: set[str], ) -> Optional[ExperimentSetPublicDump]: - """ - Narrow a validated experiment set to what belongs in the public dump. - - Drops calibrations an anonymous caller may not read, then experiments left with no score sets, and - returns None for an experiment set left with no experiments. The score sets themselves need no filter: - the loading query already restricts them to published, CC0-licensed ones. - - Narrowing the validated view rather than the ORM graph is deliberate. ``ExperimentSet.experiments`` and - ``ScoreSet.score_calibrations`` are both mapped with ``cascade="all, delete-orphan"``, so removing a - member from either ORM collection marks the removed row as an orphan and the next flush deletes it. - This script can flush: ``with_database_session`` commits when invoked with ``--commit``. - - Args: - experiment_set_view (ExperimentSetPublicDump): The validated experiment set to narrow. - visible_calibration_ids (set[int]): Ids of the calibrations an anonymous caller may read. - - Returns: - Optional[ExperimentSetPublicDump]: The narrowed experiment set, or None if nothing public remains. + """Narrow a validated experiment set to what belongs in the public dump. + + Drops calibrations the caller may not READ, blanks a superseding score set the caller may not READ, + drops experiments left with no score sets, and returns None if no experiments remain. The score sets + themselves need no filter — the loading query already restricts them to published, CC0-licensed rows. + + Blanking supersession matters because a published score set is often superseded by a *private* + in-progress replacement; without it, `main.json` would name an unreleased score set's URN and title + in an archive published to Zenodo, which can't be recalled. + + `readable_superseding_urns` gates on READ, the rule `_score_set_response` applies. It is + defense-in-depth rather than the live gate, and it can only subtract: `published_experiment_sets` + loads members through a published+CC0 filter that propagates to `superseding_score_set`, so a + private replacement is already `None` before this runs — and so is a *published* successor whose + license keeps it out of the archive. That second case is unresolved: the consumer reads a superseded + score set as current. Naming it would need the successor fetched by a separate unfiltered query and + injected into the view, not merely left un-blanked; whether it should be named is an open policy + question. Both behaviors are pinned by + `test_supersession_is_named_only_when_the_archive_carries_the_successor`. + + Narrows the validated view rather than the ORM graph because `ExperimentSet.experiments` and + `ScoreSet.score_calibrations` cascade-delete orphans, and this script can flush (`--commit`). """ experiments = [] for experiment_view in experiment_set_view.experiments: @@ -261,7 +315,13 @@ def public_experiment_set( calibration for calibration in (score_set_view.score_calibrations or []) if calibration.id in visible_calibration_ids - ] + ], + "superseding_score_set": ( + score_set_view.superseding_score_set + if score_set_view.superseding_score_set is not None + and score_set_view.superseding_score_set.urn in readable_superseding_urns + else None + ), } ) for score_set_view in experiment_view.score_sets @@ -275,11 +335,11 @@ def public_experiment_set( def published_experiment_sets(db: Session) -> list[ExperimentSet]: - """Every published experiment set, with its members narrowed to what the dump may carry. + """Every published experiment set, with members narrowed to what the dump may carry. - The narrowing is in the loader rather than applied afterwards, so an unpublished experiment or a - non-CC0 score set is never loaded onto the graph the metadata view is validated from. An experiment - set can survive this with no members left; ``public_experiment_set`` drops those. + Narrowing happens in the loader so an unpublished experiment or non-CC0 score set is never loaded + onto the graph the metadata view validates from. An experiment set can still end up with no members + left; `public_experiment_set` drops those. """ return list( db.scalars( @@ -304,13 +364,11 @@ def published_experiment_sets(db: Session) -> list[ExperimentSet]: def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[str]]: """The `main.json` payload, and the score-set URNs whose artifacts the archive carries. - One function for both because they are one decision: a score set is in the archive exactly when its - metadata survived narrowing, so deriving the URN list from the narrowed views rather than from the - query keeps the two from disagreeing. + One function for both: a score set is in the archive exactly when its metadata survives narrowing, + so deriving the URN list from the narrowed views (not the query) keeps the two in sync. - Publishing a score set does not publish its calibrations — a calibration keeps its own `private` flag - and a stricter READ rule — so which calibrations appear is asked of *principal* rather than inferred - from the score set's own visibility. + Calibration visibility is asked of *principal* rather than inferred from score-set visibility, since + publishing a score set doesn't publish its calibrations. """ experiment_sets = published_experiment_sets(db) @@ -333,10 +391,24 @@ def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[ # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. + # Asked of the ORM graph, not the validated views: a successor is typically outside the loading + # query's published+CC0 filter, so checking the view would miss it. READ is the gate, but this set + # only ever subtracts — the loader nulls a non-CC0 successor on the view first, so being readable + # here is not enough to get one named. See `public_experiment_set`. + readable_superseding_urns: set[str] = { + str(score_set_orm.superseding_score_set.urn) + for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) + if score_set_orm.superseding_score_set is not None + and score_set_orm.superseding_score_set.urn is not None + and has_permission(principal.user_data, score_set_orm.superseding_score_set, Action.READ).permitted + } + experiment_set_views = [ narrowed for narrowed in ( - public_experiment_set(ExperimentSetPublicDump.model_validate(es), visible_calibration_ids) + public_experiment_set( + ExperimentSetPublicDump.model_validate(es), visible_calibration_ids, readable_superseding_urns + ) for es in experiment_sets ) if narrowed is not None @@ -360,15 +432,12 @@ def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[ def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> list[str]: """Write every member of the public dump into *archive*, and report the score sets carried. - Takes the archive rather than a filename so the whole composition — metadata, resources, and each - score set's artifacts — can be exercised without touching the filesystem. + Takes the archive rather than a filename so the whole composition can be exercised without touching + the filesystem. """ metadata, score_set_urns = public_dump_metadata(db, principal) - - # Metadata for all data sets goes in a single JSON file. archive.writestr("main.json", json.dumps(jsonable_encoder(metadata))) - # Copy the CC0 license and README. resources_dir = os.path.join(os.path.dirname(__file__), "resources") archive.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") archive.write(os.path.join(resources_dir, "README.md"), "README.md") @@ -378,8 +447,8 @@ def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> li for i, score_set_urn in enumerate(score_set_urns): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() if score_set is None: - # `main.json` already names this score set, so skipping it silently would leave the archive - # advertising files it does not contain. Reachable only if the row disappears mid-run. + # main.json already names this score set, so skip-silently would advertise files it doesn't + # contain. Reachable only if the row disappears mid-run. logger.warning( f"[{i + 1}/{num_score_sets}] {score_set_urn} is named in main.json but could no longer be " "loaded; the archive will carry no files for it." diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py index 792469850..6ba5bf604 100644 --- a/src/mavedb/scripts/export_sweep.py +++ b/src/mavedb/scripts/export_sweep.py @@ -14,7 +14,7 @@ Writes one CSV row per attempted (score set, surface) pair, successes included: a report showing only failures cannot distinguish "nothing broke" from "nothing ran". Exits non-zero if any pair failed. -Every current mapped variant of every published score set is attempted. The measured cost of sweeping all +Every annotatable variant of every published score set is attempted. The measured cost of sweeping all score set level surfaces at the current database size is well under an hour for CSV surfaces and a little over an hour for VA annotation surfaces. @@ -43,13 +43,13 @@ variant_study_result, ) from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation +from mavedb.lib.annotation.context import VariantAnnotationContext, variant_annotation_context from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.score_sets import get_annotatable_variants from mavedb.lib.urns import variant_urn_sort_key -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.scripts.environment import script_environment, with_database_session @@ -78,7 +78,7 @@ ] -def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], Optional[Any]]]]: +def _surfaces(principal: Principal) -> list[tuple[str, Callable[[VariantAnnotationContext], Optional[Any]]]]: """The three annotation surfaces the API streams, named as their endpoint path segments.""" return [ ("study-result", variant_study_result), @@ -89,7 +89,7 @@ def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], @dataclass class SurfaceResult: - """What one annotation surface did across every current mapped variant of one score set.""" + """What one annotation surface did across every annotatable variant of one score set.""" score_set_urn: str surface: str @@ -101,7 +101,7 @@ class SurfaceResult: first_failing_variant_urn: str = "" exception_class: str = "" message: str = "" - #: Set when the pair was never attempted, e.g. the score set has no current mapped variants. + #: Set when the pair was never attempted, e.g. the score set has no annotatable variants. skip_reason: str = "" def record_failure(self, variant_urn: str, outcome: str, err: BaseException) -> None: @@ -145,12 +145,17 @@ class SweepTotals: def sweep_annotation_surface( + db: Session, score_set_urn: str, surface: str, - annotate: Callable[[MappedVariant], Optional[Any]], - mapped_variants: list[MappedVariant], + annotate: Callable[[VariantAnnotationContext], Optional[Any]], + variants: list[Variant], ) -> SurfaceResult: - """Attempt one annotation surface across every current mapped variant of one score set. + """Attempt one annotation surface across every annotatable variant of one score set. + + The context is built per variant per surface, rather than once and reused, because that is what the + streaming endpoints do per request — a defect in context construction is a defect in what the API + serves, and the sweep only sees it by taking the same path. Nothing raises out of here. A sweep that aborted on the first bad variant would report the corpus as far healthier than it is. @@ -158,14 +163,15 @@ def sweep_annotation_surface( result = SurfaceResult( score_set_urn=score_set_urn, surface=surface, - variants_attempted=len(mapped_variants), + variants_attempted=len(variants), ) - for mapped_variant in mapped_variants: - variant_urn = getattr(mapped_variant.variant, "urn", "") or "" + for variant in variants: + variant_urn = getattr(variant, "urn", "") or "" try: - annotation = annotate(mapped_variant) + context = variant_annotation_context(db, variant) + annotation = annotate(context) if context is not None else None except EXPECTED_ABSENCE_EXCEPTIONS: # An expected absence, drawn from the same definition the streaming endpoints use so the two # cannot disagree. Counting it as a failure would bury real defects under millions of @@ -347,7 +353,7 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st logger.info( f"Sweeping {len(score_sets)} published score sets across {len(surfaces) + 1} surfaces, " - "attempting every current mapped variant of each." + "attempting every annotatable variant of each." ) if max_score_sets is not None: logger.warning(f"Bounded to the first {max_score_sets} score sets by --max-score-sets; not full coverage.") @@ -356,21 +362,21 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st for index, score_set in enumerate(score_sets): urn = score_set.urn or f"" - mapped_variants = list(get_current_mapped_variants_for_annotation(db, score_set)) + variants = list(get_annotatable_variants(db, score_set)) variant_count = variant_counts.get(score_set.id, 0) surface_results: list[SurfaceResult] = [] # The annotation surfaces need a current mapping to have anything to say. The CSV surface does # not: it emits a row per variant and outer-joins the mapping, so an unmapped variant still gets # a row of NA columns. The two skip on different conditions for that reason. - if mapped_variants: + if variants: surface_results.extend( - sweep_annotation_surface(urn, surface, annotate, mapped_variants) for surface, annotate in surfaces + sweep_annotation_surface(db, urn, surface, annotate, variants) for surface, annotate in surfaces ) else: surface_results.extend( SurfaceResult( - score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no current mapped variants" + score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no annotatable variants" ) for surface, _ in surfaces ) @@ -382,9 +388,9 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st SurfaceResult(score_set_urn=urn, surface="score-set-csv", outcome=SKIPPED, skip_reason="no variants") ) - if mapped_variants or variant_count: + if variants or variant_count: totals.score_sets_attempted += 1 - totals.variants_attempted += len(mapped_variants) + totals.variants_attempted += len(variants) else: totals.score_sets_skipped += 1 @@ -418,7 +424,7 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st logger.info(f"Wrote {len(rows)} rows to {output_path}") logger.info( f"Score sets: {totals.score_sets_published} published, {totals.score_sets_attempted} attempted, " - f"{totals.score_sets_skipped} skipped with no current mapped variants." + f"{totals.score_sets_skipped} skipped with no annotatable variants." ) logger.info(f"Variants: {totals.variants_attempted} attempted per surface, every one held by those score sets.") logger.info("Outcomes: " + ", ".join(f"{outcome}={count}" for outcome, count in sorted(outcomes.items()))) diff --git a/src/mavedb/scripts/pipeline_tracking.py b/src/mavedb/scripts/pipeline_tracking.py new file mode 100644 index 000000000..15f7c4441 --- /dev/null +++ b/src/mavedb/scripts/pipeline_tracking.py @@ -0,0 +1,195 @@ +"""Operator-facing CLI for tracking which score sets need a pipeline run. + +Command +------- +list-score-sets + Produces a table of all score sets with their most recent pipeline run and + whether they need to be re-processed since a given deployment cutoff date. + +Usage: + + # Tracking list — show all published score sets and last pipeline run + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets + + # Filter to only those whose last run is before (or missing since) a deployment date + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets \\ + --needs-rerun-since 2026-05-01 + + # Include private score sets + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --include-private + + # JSON output for piping / spreadsheet import + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --json +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +import asyncclick as click +from sqlalchemy import Integer, cast, select +from sqlalchemy.orm import Session + +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + + +def _format_dt(dt: Optional[datetime]) -> str: + return dt.isoformat() if dt else "-" + + +def _needs_rerun(last_pipeline_finished: Optional[datetime], cutoff: Optional[datetime]) -> bool: + """Return True if the score set needs a re-run relative to *cutoff*.""" + if cutoff is None: + return False + if last_pipeline_finished is None: + return True + + # Normalise both to UTC-aware for comparison + aware_cutoff = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=timezone.utc) + aware_finished = ( + last_pipeline_finished if last_pipeline_finished.tzinfo else last_pipeline_finished.replace(tzinfo=timezone.utc) + ) + return aware_finished < aware_cutoff + + +def _last_pipeline_subquery(db: Session, score_set_id: int) -> Optional[Pipeline]: + """Return the most recently finished (or created) pipeline for *score_set_id*.""" + # score_set_id is stored as an integer in job_params JSONB + job_run_sq = ( + select(JobRun.pipeline_id) + .where(cast(JobRun.job_params["score_set_id"].astext, Integer) == score_set_id) + .where(JobRun.pipeline_id.is_not(None)) + .distinct() + .subquery() + ) + + return db.scalars( + select(Pipeline).where(Pipeline.id.in_(select(job_run_sq))).order_by(Pipeline.created_at.desc()).limit(1) + ).one_or_none() + + +def _build_score_set_rows( + db: Session, + score_sets: list[ScoreSet], + cutoff: Optional[datetime], +) -> list[dict]: + rows = [] + for ss in score_sets: + last_pipeline = _last_pipeline_subquery(db, ss.id) + last_pipeline_name = last_pipeline.name if last_pipeline else None + last_pipeline_status = str(last_pipeline.status) if last_pipeline else None + last_finished = last_pipeline.finished_at if last_pipeline else None + last_created = last_pipeline.created_at if last_pipeline else None + + rows.append( + { + "score_set_urn": ss.urn or "(no urn)", + "processing_state": str(ss.processing_state) if ss.processing_state else "-", + "mapping_state": str(ss.mapping_state) if ss.mapping_state else "-", + "num_variants": ss.num_variants, + "private": ss.private, + "last_pipeline_name": last_pipeline_name or "-", + "last_pipeline_status": last_pipeline_status or "-", + "last_pipeline_created_at": _format_dt(last_created), + "last_pipeline_finished_at": _format_dt(last_finished), + "needs_rerun": _needs_rerun(last_finished, cutoff), + } + ) + + return rows + + +@script_environment.command(name="list-score-sets") +@with_database_session +@click.option( + "--needs-rerun-since", + "needs_rerun_since", + default=None, + help=( + "ISO date/datetime of a deployment cutoff (e.g. 2026-05-01 or 2026-05-01T12:00:00). " + "Score sets whose last pipeline finished before this timestamp (or have never run) are " + "flagged as needing a re-run." + ), +) +@click.option("--include-private", is_flag=True, default=False, help="Include private (unpublished) score sets.") +@click.option("--needs-rerun-only", is_flag=True, default=False, help="Only show score sets that need a re-run.") +@click.option("--limit", type=int, default=None, help="Cap the number of rows returned (applied after all filtering).") +@click.option("--json", "as_json", is_flag=True, help="Emit results as JSON.") +def list_score_sets( + db: Session, + needs_rerun_since: Optional[str], + include_private: bool, + needs_rerun_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """List all score sets with their last pipeline run and re-run status.""" + cutoff: Optional[datetime] = None + if needs_rerun_since: + try: + cutoff = datetime.fromisoformat(needs_rerun_since) + except ValueError: + click.echo( + f"Invalid --needs-rerun-since value: {needs_rerun_since!r}. Use ISO format e.g. 2026-05-01.", err=True + ) + raise SystemExit(1) + + query = select(ScoreSet).where(ScoreSet.urn.is_not(None)) + if not include_private: + query = query.where(ScoreSet.private == False) # noqa: E712 + query = query.order_by(ScoreSet.urn) + + score_sets = db.scalars(query).all() + rows = _build_score_set_rows(db, list(score_sets), cutoff) + + if needs_rerun_only: + rows = [r for r in rows if r["needs_rerun"]] + + if limit is not None: + rows = rows[:limit] + + if as_json: + click.echo(json.dumps(rows, indent=2, default=str)) + return + + if not rows: + click.echo("No score sets match the given filters.") + return + + needs_rerun_count = sum(1 for r in rows if r["needs_rerun"]) + click.echo(f"Total: {len(rows)} score set(s)" + (f", {needs_rerun_count} need re-run" if cutoff else "")) + if cutoff: + click.echo(f"Cutoff: {cutoff.isoformat()}") + click.echo() + + col_w = {"urn": 28, "ps": 28, "ms": 26, "nv": 8, "pipe": 28, "pipe_status": 12, "finished": 12} + header = ( + f"{'URN':<{col_w['urn']}} {'PROC_STATE':<{col_w['ps']}} {'MAP_STATE':<{col_w['ms']}} " + f"{'VARIANTS':>{col_w['nv']}} {'LAST_PIPELINE':<{col_w['pipe']}} " + f"{'PIPE_STATUS':<{col_w['pipe_status']}} {'LAST_FINISHED':<{col_w['finished']}}" + + (" RERUN?" if cutoff else "") + ) + click.echo(header) + click.echo("-" * len(header)) + + for r in rows: + rerun_flag = ("YES" if r["needs_rerun"] else "no") if cutoff else "" + click.echo( + f"{r['score_set_urn']:<{col_w['urn']}} " + f"{r['processing_state']:<{col_w['ps']}} " + f"{r['mapping_state']:<{col_w['ms']}} " + f"{r['num_variants']:>{col_w['nv']}} " + f"{r['last_pipeline_name']:<{col_w['pipe']}} " + f"{r['last_pipeline_status']:<{col_w['pipe_status']}} " + f"{r['last_pipeline_finished_at']:<{col_w['finished']}}" + (f" {rerun_flag}" if cutoff else "") + ) + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md index 3f441fc0c..792b245a4 100644 --- a/src/mavedb/scripts/resources/CHANGELOG.md +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -10,6 +10,42 @@ given archive was generated. --- +## [Unreleased] + +### Added + +- **VRS objects** — `vrs/{urn}.vrs.ndjson` for every score set that has completed + the mapping pipeline. One record per variant placed on a reference, carrying the `pre_mapped` / + `post_mapped` GA4GH VRS pair and the Cat-VRS `categorical_variant`. Nested objects + cannot live in a CSV cell, so they get their own artifact; join to + `csv/{urn}.annotations.csv` on `mavedb.post_mapped_vrs_id`. No part of this file is + withheld on score-calibration visibility grounds, because none of it is derived + from a calibration. + +### Deprecated + +- **`mapped/{urn}.mapped-variants.json`** — superseded by `vrs/{urn}.vrs.ndjson`. It + is sourced from the pre-allele-graph mapping store, which no longer receives + writes, so it now appears only for score sets mapped before that migration. It will + be removed in a future version. + +### Changed + +- gnomAD frequencies are now reported from whichever release each allele's current + record came from, rather than only from the single release the deployment serves. + An allele not covered by a newer release previously reported `NA` for every gnomAD + column; it now reports the frequency MaveDB actually holds, with + `gnomad.gnomad_version` naming its release. Expect a mixture of releases within one + file and read that column per row. + +### Fixed + +- The three mapping-derived artifacts are no longer omitted for score sets mapped + after the allele-graph migration. +- `main.json` no longer names an unpublished superseding score set's URN and title. + +--- + ## [5] — 2026-06-25 Version 5 is the first version tracked in this changelog. Versions 1–4 contained @@ -28,7 +64,7 @@ outputs derived from the MaveDB variant mapping pipeline. VRS schema version, the mapping API version, and the ClinGen allele ID, mirroring `GET /api/v1/score-sets/{urn}/mapped-variants`. - **VA-Spec annotation NDJSON** — `va/{urn}.va.ndjson` for every mapped score set. - One line per current mapped variant carrying its highest materialized GA4GH + One line per variant currently placed on a reference, carrying its highest materialized GA4GH VA-Spec layer (study result → functional-impact statement → pathogenicity statement), mirroring the `annotated-variants` streaming endpoints. - **ClinVar release columns** in the annotation CSVs, covering the `2015_02` diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index a010be4a5..aab9e7d5f 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -38,11 +38,14 @@ mavedb-dump.YYYYMMDDHHMMSS.zip │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) │ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) -├── mapped/ -│ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS +├── vrs/ +│ └── {urn}.vrs.ndjson # GA4GH VRS and Cat-VRS objects, one record per variant placed on a reference │ # (score sets that have completed mapping only) +├── mapped/ +│ └── {urn}.mapped-variants.json # Legacy; superseded by vrs/. Score sets mapped before the +│ # allele-graph migration only └── va/ - └── {urn}.va.ndjson # GA4GH VA-Spec annotations, one record per mapped variant + └── {urn}.va.ndjson # GA4GH VA-Spec annotations, one record per variant placed on a reference # (score sets that have completed mapping only) ``` @@ -137,7 +140,7 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `gnomad.gnomad_faf95_max` | Maximum filtering allele frequency at 95% confidence across genetic ancestry groups | | `gnomad.gnomad_faf95_max_ancestry` | Genetic ancestry group attaining `gnomad_faf95_max` | | `gnomad.gnomad_id` | gnomAD variant identifier (`chrom-pos-ref-alt`), e.g. `17-43092919-G-A` | -| `gnomad.gnomad_version` | gnomAD release the frequencies were drawn from (e.g. `v4.1`) | +| `gnomad.gnomad_version` | gnomAD release *this row's* frequencies were drawn from (e.g. `v4.1`). Varies by row — see Caveats | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. @@ -183,11 +186,44 @@ Variants that could not be mapped, or for which a specific annotation is unavail will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in a future release. +### `vrs/{urn}.vrs.ndjson` + +[Newline-delimited JSON](https://ndjson.org/): one line per variant the pipeline placed onto a reference +sequence, carrying the GA4GH molecular objects for that variant and nothing else. Each line is: + +| Field | Description | +|-------|-------------| +| `variant_urn` | URN of the source variant — use this to join with `accession` in the CSV files | +| `pre_mapped` | VRS object using coordinates on the assay's own reference sequence (transcript or protein accession) | +| `post_mapped` | VRS object for the measured allele, lifted to a genomic or transcript reference | +| `categorical_variant` | GA4GH Cat-VRS `CategoricalVariant`, anchored on the measured allele with the derived alleles as members | + +A separate artifact rather than columns in `csv/{urn}.annotations.csv` because these are nested objects: a +CSV cell can carry an identifier or an HGVS string, but not a VRS object. Join the two on +`mavedb.post_mapped_vrs_id`, which is the same identifier as `post_mapped.id`. + +`categorical_variant` is spec-pure — it contains no MaveDB-specific fields. A variant the pipeline could +not place carries no reference coordinates and is omitted from this file, so every line carries a non-null +`post_mapped` and the line count equals the number of variants with reference coordinates. `pre_mapped` is +`null` where no assay-frame VRS was recorded, and `categorical_variant` is `null` where the stored VRS +could not be resolved to a GA4GH object. + +Unlike `va/{urn}.va.ndjson`, nothing here is derived from a score calibration, so no part of this file is +withheld on calibration-visibility grounds. + +**Only present for score sets that have completed the MaveDB mapping pipeline.** + +--- + ### `mapped/{urn}.mapped-variants.json` -A JSON array of the score set's **current** mapped variant records. The same shape as -`GET /api/v1/score-sets/{urn}/mapped-variants`, narrowed to `current: true`. Each record -corresponds to a single variant: +> **Legacy artifact, superseded by `vrs/{urn}.vrs.ndjson`.** This file is sourced from the +> pre-allele-graph mapping store, which no longer receives writes, so it is present only for score sets +> mapped before that migration and will be removed in a future release. `vrs/{urn}.vrs.ndjson` carries the +> same `preMapped`/`postMapped` pair for *every* mapped score set, plus the Cat-VRS object. New consumers +> should read that instead. + +A JSON array of the score set's **current** mapped variant records, one per variant: | Field | Description | |-------|-------------| @@ -205,14 +241,15 @@ corresponds to a single variant: `preMapped` and `postMapped` are raw GA4GH VRS objects (JSON). The `type` field within them may be `"Allele"`, `"Haplotype"`, or `"CisPhasedBlock"` depending on the variant. Records where mapping failed will have `preMapped: null`, `postMapped: null`, and a non-null `errorMessage`. **Only -present for score sets that have completed the MaveDB mapping pipeline.** +present for score sets mapped before the allele-graph migration** — see the note above. --- ### `va/{urn}.va.ndjson` -[Newline-delimited JSON](https://ndjson.org/): one line per current mapped variant. Each line is an -envelope mirroring the `GET /api/v1/score-sets/{urn}/annotated-variants/*` streaming endpoints: +[Newline-delimited JSON](https://ndjson.org/): one line per variant the pipeline currently places onto a +reference sequence. Each line is an envelope mirroring the +`GET /api/v1/score-sets/{urn}/annotated-variants/*` streaming endpoints: ```json {"variant_urn": "urn:mavedb:00000001-a-1#1", "annotation": { ... }} @@ -245,10 +282,11 @@ be combined with other evidence downstream, not as a standalone clinical verdict Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which includes them under a `research_use_only` flag. -`annotation` is `null` for current mapped variants that have no post-mapped allele (and therefore -cannot be annotated); the `variant_urn` is still present on those lines. Every current mapped variant -produces exactly one line, so the line count equals the current mapped-variant count. **Only present -for score sets that have completed the MaveDB mapping pipeline.** +`annotation` is `null` where no VA-Spec layer could be built for a variant that does have reference +coordinates. A variant the pipeline could not place has none, and is omitted from this file entirely. +Every variant it currently places produces exactly one line, so the line count equals the number of +variants with current reference coordinates. **Only present for score sets that have completed the MaveDB +mapping pipeline.** --- @@ -296,7 +334,7 @@ score_set = next( - Only **published**, **CC0-licensed** data is included. Datasets with other licenses are not present in this dump even if they are publicly visible on MaveDB. -- Annotation files (`.annotations.csv`), mapped variant files (`.mapped-variants.json`), and +- Annotation files (`.annotations.csv`), the legacy `.mapped-variants.json` files, and VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. @@ -312,12 +350,16 @@ score_set = next( `preMapped: null` / `postMapped: null` in the JSON. - This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` JSON files include only each variant's current mapping — superseded records from earlier mapping - runs are not retained here. (Unlike this per-run dump, `GET /api/v1/score-sets/{urn}/mapped-variants` - does return superseded mappings, for callers that need that history directly.) The same applies to + runs are not retained here. (Callers that need mapping history directly can ask the API for a past + instant with the `as_of` parameter.) The same applies to `annotations.csv` and the `va/` files: annotations are always reported with respect to the current mapping object. If you need MaveDB's state as of a specific point in time, use a dump from that time (e.g. an earlier Zenodo-archived release) rather than looking for historical rows within one. -- gnomAD allele frequencies in `annotations.csv` are sourced from **gnomAD v4.1** specifically. +- gnomAD allele frequencies are **not all from one release**. MaveDB records a frequency per allele from + whichever gnomAD release last covered it, and a release that does not include an allele leaves that + allele's earlier frequency in place. Read `gnomad.gnomad_version` per row rather than assuming a single + release for the file; a mixture is normal and expected. (ClinVar differs: each release MaveDB holds gets + its own `clinvar.YYYY_MM` column group, so those coexist as columns rather than varying by row.) - `preMapped` VRS objects reference the assay's input sequence (a transcript or protein accession). `postMapped` VRS objects are remapped to the **GRCh38** reference genome. Do not compare coordinates between `preMapped` and `postMapped` directly. diff --git a/src/mavedb/scripts/run_pipeline.py b/src/mavedb/scripts/run_pipeline.py index c947fe122..658037fbb 100644 --- a/src/mavedb/scripts/run_pipeline.py +++ b/src/mavedb/scripts/run_pipeline.py @@ -11,7 +11,6 @@ poetry run python -m mavedb.scripts.run_pipeline --list """ -import datetime import logging import sys @@ -21,10 +20,9 @@ from mavedb.db.session import SessionLocal from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from mavedb.worker.lib.managers.utils import arq_job_id from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -108,40 +106,25 @@ async def main( click.echo(f"User not found: {resolved_updater_id}", err=True) sys.exit(1) - correlation_id = f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" - pipeline_params: dict = { - "correlation_id": correlation_id, - "score_set_id": score_set.id, - "updater_id": user.id, - } - for key, value in extra_params: - pipeline_params[key] = value - - try: - pipeline_factory = PipelineFactory(session=db) - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=user, - pipeline_params=pipeline_params, - ) - except (KeyError, ValueError) as e: - click.echo(f"Failed to create pipeline: {e}", err=True) - sys.exit(1) - - click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={correlation_id})") - # Connect to Redis and enqueue redis = await create_pool(RedisWorkerSettings) try: - job = await redis.enqueue_job( - pipeline_entrypoint.job_function, - pipeline_entrypoint.id, - _job_id=arq_job_id(pipeline_entrypoint), + pipeline, job = await enqueue_pipeline_for_score_set( + db, + redis, + pipeline_name=pipeline_name, + score_set=score_set, + user=user, + extra_params={key: value for key, value in extra_params}, ) + click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={pipeline.correlation_id})") if job: click.echo(f"Enqueued start_pipeline job: {job.job_id}. Pipeline will execute asynchronously.") else: click.echo("Job was already enqueued (duplicate).", err=True) + except (KeyError, ValueError) as e: + click.echo(f"Failed to create pipeline: {e}", err=True) + sys.exit(1) finally: await redis.aclose() db.close() diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index bfe896eea..01770253e 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -84,8 +84,6 @@ { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } diff --git a/src/mavedb/scripts/variant_annotations.py b/src/mavedb/scripts/variant_annotations.py index ac9ff916c..fe19172c0 100644 --- a/src/mavedb/scripts/variant_annotations.py +++ b/src/mavedb/scripts/variant_annotations.py @@ -1,14 +1,17 @@ -"""Operator-facing CLI for inspecting variant annotation state. +"""Operator-facing CLI for inspecting a score set's current annotation status. + +The score set is the entry point (pipelines run per score set), but annotation status is a per-allele +fact: this resolves the score set's *current* alleles (and variants) through the live mapping links and +reports each subject's true current status — including statuses produced by another score set's run +that landed on a shared allele. It does **not** filter by which run wrote the event. Usage: - # Summarize annotation status counts for all variants in a score set poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 --json -For per-variant detail, query the v_variant_annotations view directly: - SELECT * FROM v_variant_annotations WHERE score_set_urn = 'urn:mavedb:00000001-a-1'; - SELECT * FROM v_variant_annotations WHERE variant_urn = 'urn:mavedb:00000001-a-1#42'; - SELECT * FROM v_variant_annotations WHERE annotation_status = 'failed'; +For per-subject detail, query the v_current_annotation_events view directly: + SELECT * FROM v_current_annotation_events WHERE allele_id = ; + SELECT * FROM v_current_annotation_events WHERE disposition = 'failed'; """ import json @@ -16,12 +19,13 @@ from typing import Optional import asyncclick as click -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session +from mavedb.lib.clingen.alleles import get_alleles_for_score_set +from mavedb.models.annotation_event_view import CurrentAnnotationEventView from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -31,34 +35,52 @@ def _get_score_set(db: Session, urn: str) -> Optional[ScoreSet]: return db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() +def current_annotation_summary(db: Session, score_set: ScoreSet) -> list[dict]: + """Current annotation disposition counts for a score set, by annotation type. + + Resolves the score set's subjects — its current alleles (via the live mapping links) for + allele-subject types, and its variants for variant-subject types (mapping/RT/LDH) — then counts the + current status of each in ``v_current_annotation_events``. Allele-subject counts are per allele + (shared alleles counted once), reflecting the true current status regardless of which run produced + it; variant-subject counts are per variant. + """ + allele_ids = {row.allele_id for row in get_alleles_for_score_set(db, score_set.id)} + variant_ids = set(db.scalars(select(Variant.id).where(Variant.score_set_id == score_set.id)).all()) + + if not allele_ids and not variant_ids: + return [] + + rows = db.execute( + select( + CurrentAnnotationEventView.annotation_type, + CurrentAnnotationEventView.disposition, + func.count().label("count"), + ) + .where( + or_( + CurrentAnnotationEventView.allele_id.in_(allele_ids), + CurrentAnnotationEventView.variant_id.in_(variant_ids), + ) + ) + .group_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + .order_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + ).all() + + return [{"annotation_type": r.annotation_type, "disposition": r.disposition, "count": r.count} for r in rows] + + @script_environment.command(name="show-score-set") @with_database_session @click.argument("score_set_urn") @click.option("--json", "as_json", is_flag=True, help="Emit result as JSON.") def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: - """Summarize annotation status counts for all variants in a score set.""" + """Summarize the current annotation status of a score set's alleles and variants.""" score_set = _get_score_set(db, score_set_urn) if score_set is None: click.echo(f"Score set not found: {score_set_urn}", err=True) raise SystemExit(1) - # Count current annotation statuses grouped by annotation_type and status - rows = db.execute( - select( - VariantAnnotationStatus.annotation_type, - VariantAnnotationStatus.status, - func.count().label("count"), - ) - .join(Variant, Variant.id == VariantAnnotationStatus.variant_id) - .where( - Variant.score_set_id == score_set.id, - VariantAnnotationStatus.current == True, # noqa: E712 - ) - .group_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - ).all() - - # Total variant count for the score set + summary = current_annotation_summary(db, score_set) total_variants = db.scalar(select(func.count()).where(Variant.score_set_id == score_set.id)) or 0 if as_json: @@ -67,9 +89,7 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: { "score_set_urn": score_set_urn, "total_variants": total_variants, - "annotation_summary": [ - {"annotation_type": r.annotation_type, "status": r.status, "count": r.count} for r in rows - ], + "annotation_summary": summary, }, indent=2, ) @@ -77,12 +97,12 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: return click.echo(f"Score set: {score_set_urn} ({total_variants} variants)") - click.echo(f"\n{'ANNOTATION TYPE':<32} {'STATUS':<10} COUNT") - for r in rows: - click.echo(f"{r.annotation_type:<32} {str(r.status):<10} {r.count}") + click.echo(f"\n{'ANNOTATION TYPE':<32} {'DISPOSITION':<16} COUNT") + for r in summary: + click.echo(f"{r['annotation_type']:<32} {str(r['disposition']):<16} {r['count']}") - if not rows: - click.echo("No annotation status records found.") + if not summary: + click.echo("No current annotation events found for this score set.") if __name__ == "__main__": diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 880bfcfe1..9cffb81d2 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -40,8 +40,10 @@ from mavedb.models import * # noqa: F403 from mavedb.routers import ( access_keys, + alleles, alphafold, api_information, + clingen_alleles, collections, controlled_keywords, doi_identifiers, @@ -97,7 +99,9 @@ ) app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) +app.include_router(alleles.router) app.include_router(api_information.router) +app.include_router(clingen_alleles.router) app.include_router(collections.router) app.include_router(controlled_keywords.router) app.include_router(doi_identifiers.router) @@ -245,7 +249,9 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, + alleles.metadata, api_information.metadata, + clingen_alleles.metadata, collections.metadata, controlled_keywords.metadata, doi_identifiers.metadata, @@ -255,7 +261,6 @@ def customize_openapi_schema(): hgvs.metadata, licenses.metadata, # log.metadata, - mapped_variant.metadata, orcid.metadata, permissions.metadata, publication_identifiers.metadata, diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py new file mode 100644 index 000000000..a995fabcb --- /dev/null +++ b/src/mavedb/view_models/allele_annotation.py @@ -0,0 +1,60 @@ +"""Response view models for an allele's external annotations (VEP / gnomAD / ClinVar). + +The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses +(``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and alongside the cross-layer +equivalence class on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather +than in either one's file. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class VepAnnotation(BaseModel): + """VEP most-severe functional consequence and the Ensembl release it resolved under.""" + + consequence: Optional[str] = None + source_version: str + + class Config: + from_attributes = True + + +class GnomadAnnotation(BaseModel): + """gnomAD population frequency for an allele.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] = None + db_version: str + db_identifier: str + + class Config: + from_attributes = True + + +class ClinvarAnnotation(BaseModel): + """One ClinVar assertion for an allele (an allele may carry one per release).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] = None + clinvar_allele_id: str + db_version: str + + class Config: + from_attributes = True + + +class AlleleAnnotations(BaseModel): + """The external annotations for one allele, sparse — each source absent unless it has data.""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = [] + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_detail.py b/src/mavedb/view_models/allele_detail.py new file mode 100644 index 000000000..7a6fabe5b --- /dev/null +++ b/src/mavedb/view_models/allele_detail.py @@ -0,0 +1,32 @@ +"""Response view models for the allele-detail view (``GET /alleles/{digest|CAID}``). + +Pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses; the +allele-grain counterpart of ``view_models.variant_detail``. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel + + +class AlleleDetail(BaseModel): + """The allele-detail envelope (``GET /alleles/{digest|CAID}``). + + ``alleles`` is the full cross-layer equivalence class, keyed by VRS digest; ``isFocus`` marks + the queried allele. ``annotations`` shares those same keys. Measurement-agnostic: no score, + classification, or re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). + """ + + digest: str + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + vrs: Optional[dict[str, Any]] = None + + alleles: dict[str, AlleleIdentity] = {} + annotations: dict[str, AlleleAnnotations] = {} + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_identity.py b/src/mavedb/view_models/allele_identity.py new file mode 100644 index 000000000..a7faf24fc --- /dev/null +++ b/src/mavedb/view_models/allele_identity.py @@ -0,0 +1,30 @@ +"""Response view model for the shared allele molecular-identity shape. + +Pydantic boundary over :class:`lib.allele_identity.AlleleIdentity`. Shared by the ``alleles`` map +on both ``GET /variants/{urn}`` and ``GET /alleles/{digest}``. See the lib docstring for the +focus-relative semantics of each axis. +""" + +from typing import Optional + +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.view_models.base.base import BaseModel + + +class AlleleIdentity(BaseModel): + """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the + view's focus allele. ``isFocus`` marks the anchor; ``relation`` and ``derivation`` describe + every other member's relationship to it and are absent on the focus itself. + """ + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + is_focus: bool + relation: Optional[str] = None + derivation: Optional[AlleleDerivation] = None + # VRS digest of this allele's c<->g projection counterpart, when one exists. + projection_of: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py new file mode 100644 index 000000000..0baebaddd --- /dev/null +++ b/src/mavedb/view_models/allele_measurement.py @@ -0,0 +1,43 @@ +"""Response view model for the variant page's measurements list +(``GET /clingen-alleles/{caid}/measurements``). + +The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass +(``from_attributes`` coerces it directly); aliases camelize per the shared base config. +""" + +from typing import Optional + +from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class AlleleMeasurement(BaseModel): + """One measurement in the queried ClinGen allele's cross-layer equivalence class. + + ``assayLevel`` is the level at which this measurement was actually assayed (``protein`` / ``cdna`` / + ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. + ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at + this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the + readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted + when absent or gated. ``isCurrent`` / + ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) + self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. + """ + + variant_urn: str + score: Optional[float] = None + assay_level: Optional[SequenceLevel] = None + relationship: MeasurementRelationship + assay_level_hgvs: Optional[str] = None + submitted_hgvs: Optional[str] = None + score_set_urn: str + score_set_title: str + preferred_classification: Optional[SavedFunctionalClassification] = None + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/clinical_control.py b/src/mavedb/view_models/clinical_control.py index 85cd78348..7e6a17d06 100644 --- a/src/mavedb/view_models/clinical_control.py +++ b/src/mavedb/view_models/clinical_control.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariantCreate, MappedVariantForClinicalControl + from mavedb.view_models.mapped_variant import MappedVariantCreate class ClinicalControlBase(BaseModel): @@ -41,8 +41,17 @@ class Config: from_attributes = True -class SavedClinicalControlWithMappedVariants(SavedClinicalControl): - mapped_variants: Sequence["MappedVariantForClinicalControl"] +class ClinvarVariantLink(BaseModel): + """One score-set variant a ClinVar control reaches, tagged with the annotated allele's digest.""" + + variant_urn: str + # VRS digest of the allele this control annotates. Many controls + # may share the same variant_urn, but each will have a different allele_digest. + allele_digest: Optional[str] = None + + +class SavedClinicalControlWithClinvarLinks(SavedClinicalControl): + clinvar_links: Sequence[ClinvarVariantLink] # Properties to return to non-admin clients @@ -50,7 +59,7 @@ class ClinicalControl(SavedClinicalControl): pass -class ClinicalControlWithMappedVariants(SavedClinicalControlWithMappedVariants): +class ClinicalControlWithClinvarLinks(SavedClinicalControlWithClinvarLinks): pass diff --git a/src/mavedb/view_models/gnomad_variant.py b/src/mavedb/view_models/gnomad_variant.py index 97dd675e6..116a4e21b 100644 --- a/src/mavedb/view_models/gnomad_variant.py +++ b/src/mavedb/view_models/gnomad_variant.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariant, MappedVariantCreate, SavedMappedVariant + from mavedb.view_models.mapped_variant import MappedVariantCreate class GnomADVariantBase(BaseModel): @@ -54,10 +54,21 @@ class Config: from_attributes = True -class SavedGnomADVariantWithMappedVariants(SavedGnomADVariant): - """Saved GnomAD variant records with mapped variants.""" +class GnomadVariantLink(BaseModel): + """One score-set variant a gnomAD frequency record reaches, tagged with the annotated allele's digest. - mapped_variants: Sequence["SavedMappedVariant"] + Mirrors :class:`clinical_control.ClinvarVariantLink`: a gnomAD variant fans out to every allele that + resolved to it, and each allele belongs to a score-set variant. + """ + + variant_urn: str + allele_digest: Optional[str] = None + + +class SavedGnomADVariantWithVariantLinks(SavedGnomADVariant): + """Saved gnomAD variant paired with the score-set variants (and annotated allele digests) it links to.""" + + variant_links: Sequence[GnomadVariantLink] class GnomADVariant(SavedGnomADVariant): @@ -66,7 +77,7 @@ class GnomADVariant(SavedGnomADVariant): pass -class GnomADVariantWithMappedVariants(SavedGnomADVariantWithMappedVariants): - """GnomAD variant view model with mapped variants for non-admin clients.""" +class GnomADVariantWithVariantLinks(SavedGnomADVariantWithVariantLinks): + """GnomAD variant + its score-set variant links, for non-admin clients.""" - mapped_variants: Sequence["MappedVariant"] + pass diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py new file mode 100644 index 000000000..474484e40 --- /dev/null +++ b/src/mavedb/view_models/lean_variant.py @@ -0,0 +1,73 @@ +"""Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). + +The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` +lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` +coerce them (field names line up); aliases camelize per the shared base config, so clients see +``variantUrn`` / ``assayLevel`` / ``mapped`` etc. ``response_model_exclude_none`` drops absent fields, so a +slot with no parsed block serializes as just ``{"hgvs": "..."}`` and a null mapped level drops out of the +``mapped`` object entirely. +""" + +from typing import Optional + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.base.base import BaseModel + + +class HgvsField(BaseModel): + """An HGVS expression with its parsed substitution block riding alongside when representable. + + ``hgvs`` is always present; ``position``/``ref``/``alt`` appear only for a placeable simple + substitution (the heatmap grid) and are omitted for splice/indels/multivariants. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + class Config: + from_attributes = True + + +class MappedTriple(BaseModel): + """The mapped (reference-frame) HGVS keyed by level — the canonical projection of the measured change. + + One slot per level. A nucleotide assay populates all three (``mapped[assayLevel]`` is the measured + slot; ``cdna`` is the level-invariant search key, present even when ``assayLevel`` is ``genomic``); a + protein assay populates only ``protein`` (the ambiguous c/g fan-out is not fabricated). Null slots are + omitted under ``response_model_exclude_none``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + class Config: + from_attributes = True + + +class LeanVariant(BaseModel): + """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. + + ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry + the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``SequenceLevel`` value) naming + the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` + the level-invariant search key. Fields are omitted when null. + """ + + variant_urn: str + score: Optional[float] = None + consequence: Optional[str] = None + clingen_allele_id: Optional[str] = None + assay_level_digest: Optional[str] = None + hgvs_nt: Optional[HgvsField] = None + hgvs_pro: Optional[HgvsField] = None + hgvs_splice: Optional[HgvsField] = None + assay_level: Optional[SequenceLevel] = None + mapped: MappedTriple = MappedTriple() + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 2d4856c88..7c03fd103 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -7,7 +7,7 @@ from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel from mavedb.view_models.target_gene_mapping import SavedTargetGeneMapping, TargetGeneMapping @@ -28,7 +28,7 @@ class MappedVariantBase(BaseModel): current: bool # Per-mapping QC annotations/warnings. - alignment_level: Optional[AnnotationLayer] = None + alignment_level: Optional[SequenceLevel] = None at_mismatched_locus: Optional[bool] = None near_gap: Optional[bool] = None @@ -111,22 +111,6 @@ class MappedVariantWithControls(SavedMappedVariantWithControls): gnomad_variants: Sequence["GnomADVariant"] -class MappedVariantForClinicalControl(BaseModel): - variant_urn: str - - class Config: - from_attributes = True - - @model_validator(mode="before") - def generate_score_set_urn_list(cls, data: Any): - if not hasattr(data, "variant_urn") and hasattr(data, "variant"): - try: - data.__setattr__("variant_urn", None if not data.variant else data.variant.urn) - except AttributeError as exc: - raise ValidationError(f"Unable to create {cls.__name__} without attribute: {exc}.") # type: ignore - return data - - # ruff: noqa: E402 from mavedb.view_models.clinical_control import ClinicalControl, ClinicalControlBase, SavedClinicalControl from mavedb.view_models.gnomad_variant import GnomADVariant, GnomADVariantBase, SavedGnomADVariant diff --git a/src/mavedb/view_models/target_gene_mapping.py b/src/mavedb/view_models/target_gene_mapping.py index 13e64d1a7..535508a42 100644 --- a/src/mavedb/view_models/target_gene_mapping.py +++ b/src/mavedb/view_models/target_gene_mapping.py @@ -7,13 +7,13 @@ from datetime import date from typing import Any, Optional -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel class TargetGeneMappingBase(BaseModel): - alignment_level: AnnotationLayer + alignment_level: SequenceLevel preferred: bool = False reference_assembly: Optional[str] = None diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index b01b01832..48d4edb85 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -1,15 +1,26 @@ from datetime import date -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel -from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant +from mavedb.view_models.mapped_variant import SavedMappedVariant -if TYPE_CHECKING: - from mavedb.view_models.score_set import ScoreSet, ShortScoreSet + +class VariantVrsMatch(BaseModel): + """One variant whose mapping links an allele bearing the queried VRS identifier. + + The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id + level ride from the matched + (deduplicated) allele; the URN from the variant it is linked to. Replaces the legacy MappedVariant VRS + lookup on the new Allele substrate (#743, item 3.5). + """ + + variant_urn: str + clingen_allele_id: Optional[str] = None + vrs_id: Optional[str] = None + level: Optional[str] = None class VariantEffectMeasurementBase(BaseModel): @@ -74,39 +85,3 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): """Variant effect measurement view model returned to most clients""" pass - - -class VariantEffectMeasurementWithScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and score set""" - - score_set: "ScoreSet" - mapped_variants: list[MappedVariant] - - -class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and a limited set of score set details""" - - score_set: "ShortScoreSet" - mapped_variants: list[MappedVariant] - - -class ClingenAlleleIdVariantLookupsRequest(BaseModel): - """A request to search for variants matching a list of ClinGen allele IDs""" - - clingen_allele_ids: list[str] - - -class Variant(BaseModel): - """View model for a variant, defined by its ClinGen allele id, with associated variant effect measurements""" - - clingen_allele_id: str - variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] - - -class ClingenAlleleIdVariantLookupResponse(BaseModel): - """Response model for a variant lookup by ClinGen allele ID""" - - clingen_allele_id: str - exact_match: Optional[Variant] = None - equivalent_nt: list[Variant] = [] - equivalent_aa: list[Variant] = [] diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py new file mode 100644 index 000000000..9603bc7b3 --- /dev/null +++ b/src/mavedb/view_models/variant_detail.py @@ -0,0 +1,80 @@ +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``). + +The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses +(``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its +own module — mirroring ``lib/variant_detail.py`` and the ``lean_variant`` precedent — so +``view_models/variant.py`` stays the core measurement + lookup file. +""" + +from typing import Any, Optional + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class VariantClassification(BaseModel): + """A functional classification the variant falls into, tagged with its calibration context. + + A score set may carry several calibrations, so a variant has one classification per calibration; + ``primary`` flags the UI default. The classifications are calibration-derived but as-of-invariant + (calibrations carry no valid-time), so they are always the current calibration state. + """ + + calibration_id: int + primary: bool + classification: SavedFunctionalClassification + + class Config: + from_attributes = True + + +class VariantDetail(BaseModel): + """The assayed variant-detail envelope (``GET /variants/{urn}``). + + Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair + is a client-side toggle, no refetch; the ``preMapped``/``postMapped`` raw VRS pair lets a + VRS/bulk consumer read the assayed-level and measured VRS directly) plus the spec-pure GA4GH + ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer + rides alongside, keyed by VRS digest: the ``alleles`` identity sidecar (per-allele ``level`` / + ``hgvs`` / ``clingenAlleleId`` / ``relation`` — one entry per linked allele, sharing keys with + ``annotations``) and the ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a + superseded variant self-describe: ``supersededByScoreSet`` is the superseding *score set*'s URN, + not a variant URN. Supersession is versioned at the score-set level, and a newer version may add, + drop, or renumber variants — so there is no stable superseding-*variant* pointer to hand back; a + consumer resolves the current measurement by looking this variant up within that score set. + + Unlike most MaveDB response models, this one serializes with ``exclude_none=False`` on both routes + that emit it (``GET /variants/{urn}`` and the bulk ``GET /score-sets/{urn}/variant-details`` NDJSON + stream): the shape is a stable, self-describing envelope, so every record carries the same key set + and an unmapped variant reads ``preMapped``/``postMapped``/``molecularRepresentation`` as ``null`` + rather than dropping them. The two routes are kept in lockstep — the same object, one shape. + """ + + urn: str + scores: Optional[dict[str, Any]] = None + counts: Optional[dict[str, Any]] = None + classifications: list[VariantClassification] = [] + + assay_level: Optional[SequenceLevel] = None + target_hgvs: Optional[str] = None + reference_hgvs: Optional[str] = None + assay_level_digest: Optional[str] = None + clingen_allele_id: Optional[str] = None + + pre_mapped: Optional[dict[str, Any]] = None + post_mapped: Optional[dict[str, Any]] = None + + molecular_representation: Optional[dict[str, Any]] = None + mode: Optional[str] = None + alleles: dict[str, AlleleIdentity] = {} + + annotations: dict[str, AlleleAnnotations] = {} + + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/worker/jobs/__init__.py b/src/mavedb/worker/jobs/__init__.py index e421bbad2..488398117 100644 --- a/src/mavedb/worker/jobs/__init__.py +++ b/src/mavedb/worker/jobs/__init__.py @@ -33,11 +33,13 @@ from mavedb.worker.jobs.variant_processing.mapping import ( map_variants_for_score_set, ) +from mavedb.worker.jobs.variant_processing.reverse_translation import reverse_translate_variants_for_score_set __all__ = [ # Variant processing jobs "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", # External service integration jobs "submit_score_set_mappings_to_car", "submit_score_set_mappings_to_ldh", diff --git a/src/mavedb/worker/jobs/external_services/__init__.py b/src/mavedb/worker/jobs/external_services/__init__.py index 4537c0edd..78af92516 100644 --- a/src/mavedb/worker/jobs/external_services/__init__.py +++ b/src/mavedb/worker/jobs/external_services/__init__.py @@ -5,8 +5,6 @@ - ClinGen cache pre-warming to prevent stampede on downstream annotation jobs - UniProt for protein sequence annotation and ID mapping - gnomAD for population frequency and genomic context data -- HGVS for standardized variant nomenclature population -- Variant Translation for PA<->CA allele relationship mapping - VEP for functional consequence annotation """ @@ -18,12 +16,10 @@ from .clingen_cache import warm_clingen_cache from .clinvar import refresh_clinvar_controls from .gnomad import link_gnomad_variants -from .hgvs import populate_hgvs_for_score_set from .uniprot import ( poll_uniprot_mapping_jobs_for_score_set, submit_uniprot_mapping_jobs_for_score_set, ) -from .variant_translation import populate_variant_translations_for_score_set from .vep import populate_vep_for_score_set __all__ = [ @@ -32,8 +28,6 @@ "warm_clingen_cache", "refresh_clinvar_controls", "link_gnomad_variants", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "populate_vep_for_score_set", diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 639160ed5..d563e658c 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -3,7 +3,6 @@ This module contains jobs for submitting mapped variants to ClinGen services: - ClinGen Allele Registry (CAR) for allele registration - ClinGen Linked Data Hub (LDH) for data submission -- Variant linking and association management These jobs enable integration with the ClinGen ecosystem for clinical variant interpretation and data sharing. @@ -12,13 +11,17 @@ import asyncio import functools import logging +from collections.abc import Sequence +from dataclasses import dataclass from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, + DEFAULT_CAR_SUBMISSION_BATCH_SIZE, DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_ENDPOINT, ) @@ -28,10 +31,15 @@ ClinGenLdhService, ) from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.utils import batched from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params @@ -41,6 +49,39 @@ logger = logging.getLogger(__name__) +@dataclass +class _AlleleEntry: + post_mapped: dict | None + existing_caid: str | None + + +def _annotate_caid( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CLINGEN_ALLELE_ID event for an allele (the CAID is an allele-level fact). + + The single choke point for CAR's status writes. Provenance (which variants drove the + registration) is derived from the live links as-of the event, not fanned out here. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele_id, + disposition=disposition, + reason=reason, + metadata=meta or None, + ) + + @with_pipeline_management async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ @@ -58,11 +99,11 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager (JobManager): Manager for job lifecycle and DB operations Side Effects: - - Updates MappedVariant records with ClinGen Allele IDs + - Updates Allele records with ClinGen Allele IDs - Submits data to ClinGen Allele Registry Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele counts (submitted/registered/already-registered/failed). """ # Get the job definition we are working on job = job_manager.get_job() @@ -70,11 +111,10 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: _job_required_params = ["score_set_id", "correlation_id"] validate_job_params(_job_required_params, job) - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - # Setup initial context and progress job_manager.save_to_context( { "application": "mavedb-worker", @@ -86,7 +126,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) - # Ensure we've enabled ClinGen submission if not CLIN_GEN_SUBMISSION_ENABLED: logger.warning( msg="ClinGen submission is disabled via configuration, skipping submission of mapped variants to CAR.", @@ -95,7 +134,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.skipped(data={"reason": "ClinGen submission disabled"}) - # Check for CAR submission endpoint if not CAR_SUBMISSION_ENDPOINT: logger.warning( msg="ClinGen Allele Registry submission is disabled (no submission endpoint), unable to complete submission of mapped variants to CAR.", @@ -107,180 +145,324 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: failure_category=FailureCategory.CONFIGURATION_ERROR, ) - # Fetch mapped variants with post-mapped data for the score set - variant_post_mapped_objects = job_manager.db.execute( - select(MappedVariant.id, MappedVariant.post_mapped) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() - - # Track total variants to submit - job_manager.save_to_context({"total_variants_to_submit_car": len(variant_post_mapped_objects)}) - if not variant_post_mapped_objects: + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + job_manager.save_to_context({"total_allele_variant_pairs": len(allele_rows)}) + if not allele_rows: logger.warning( - msg="No current mapped variants with post mapped metadata were found for this score set. Skipping CAR submission.", + msg="No current alleles found for this score set. Skipping CAR submission.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"submitted_count": 0, "matched_count": 0}) + return JobExecutionOutcome.succeeded( + data={ + "submitted_allele_count": 0, + "registered_allele_count": 0, + "already_registered_allele_count": 0, + "failed_allele_count": 0, + } + ) - job_manager.update_progress( - 10, 100, f"Preparing {len(variant_post_mapped_objects)} mapped variants for CAR submission." + # Group by allele_id: one allele may serve multiple variants (cross-score-set dedup). + allele_data: dict[int, _AlleleEntry] = {} + for row in allele_rows: + if row.allele_id not in allele_data: + allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) - # Build HGVS strings for submission. Don't do duplicate submissions-- store mapped variant IDs by HGVS. - # Variants that can't produce an HGVS string are annotated as failures immediately. - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - variant_post_mapped_hgvs: dict[str, list[int]] = {} - no_hgvs_count = 0 - for mapped_variant_id, post_mapped in variant_post_mapped_objects: - hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) + # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID + # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele + # units — and they cover every allele submitted, including the RT-derived ones. Each allele has + # exactly one outcome (submitted once → one response), so these sets are disjoint by construction. + linked_allele_ids: set[int] = set() + preexisting_allele_ids: set[int] = set() + failed_allele_ids: set[int] = set() + + preexisting_alleles = [] + pending_alleles = [] + for aid, entry in allele_data.items(): + if entry.existing_caid and not force_reregister: + preexisting_alleles.append(aid) + else: + pending_alleles.append(aid) + + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. + preexisting = preexisting_alleles if not force_reregister else [] + for allele_id in preexisting: + entry = allele_data[allele_id] + + preexisting_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": entry.existing_caid}, + ) + + job_manager.update_progress(10, 100, f"Preparing {len(pending_alleles)} alleles for CAR submission.") - if not hgvs_for_post_mapped: - no_hgvs_count += 1 + # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS + # (combine_cis defaults to False); those alleles are annotated as failures immediately. + hgvs_to_allele_ids: dict[str, list[int]] = {} + for allele_id in pending_alleles: + entry = allele_data[allele_id] + hgvs = get_hgvs_from_post_mapped(entry.post_mapped) + + if hgvs: + hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) + + # TODO#780 - Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as preexisting + # rather than failing the variant. + elif entry.existing_caid: + preexisting_alleles.append(allele_id) logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant_id}. Skipping submission of this variant.", + msg=( + f"Could not construct HGVS for allele {allele_id} during force re-registration " + f"(existing CAID: {entry.existing_caid!r}). Reconfirmation skipped; existing CAID retained." + ), extra=job_manager.logging_context(), ) - - mapped_variant = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id == mapped_variant_id) - ).one() - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Could not extract a valid HGVS string from post-mapped variant data.", - "annotation_metadata": {}, - }, - current=True, + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.RECONFIRMATION_SKIPPED, + metadata={"clingen_allele_id": entry.existing_caid}, ) - continue - if hgvs_for_post_mapped in variant_post_mapped_hgvs: - variant_post_mapped_hgvs[hgvs_for_post_mapped].append(mapped_variant_id) + # No HGVS-- un-submittable. The allele cannot produce an identifier to register, + # so treat this as not_applicable rather than a failure. else: - variant_post_mapped_hgvs[hgvs_for_post_mapped] = [mapped_variant_id] + failed_allele_ids.add(allele_id) + logger.warning( + msg=f"Could not construct HGVS for allele {allele_id}. Skipping CAR submission.", + extra=job_manager.logging_context(), + ) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.NO_HGVS, + error_message="Could not extract a valid HGVS string from post-mapped allele data.", + ) - job_manager.save_to_context({"unique_variants_to_submit_car": len(variant_post_mapped_hgvs)}) - job_manager.update_progress(15, 100, "Submitting mapped variants to CAR.") + job_manager.save_to_context({"unique_hgvs_to_submit_car": len(hgvs_to_allele_ids)}) - # Do submission + # Distinct alleles actually sent to CAR this run (pending alleles that yielded HGVS). + submitted_allele_ids: set[int] = { + allele_id for allele_ids in hgvs_to_allele_ids.values() for allele_id in allele_ids + } + + def _outcome_data() -> dict[str, int]: + return { + "submitted_allele_count": len(submitted_allele_ids), + "registered_allele_count": len(linked_allele_ids), + "already_registered_allele_count": len(preexisting_allele_ids), + "failed_allele_count": len(failed_allele_ids), + } + + # All pending alleles failed HGVS extraction; annotations already written above. + if not hgvs_to_allele_ids: + annotation_manager.flush() + job_manager.db.flush() + if preexisting_allele_ids: + return JobExecutionOutcome.succeeded(data=_outcome_data()) + + return JobExecutionOutcome.failed( + reason=f"No submittable alleles for score set {score_set.urn}.", + data=_outcome_data(), + failure_category=FailureCategory.DEPENDENCY_FAILURE, + ) + + job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(variant_post_mapped_hgvs.keys()) - registered_alleles = car_service.dispatch_submissions(hgvs_list) - job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") - - # CAR returns one response per submitted HGVS in the same order (see CAR API docs). - # Zip the submissions with the responses and annotate each based on success or error. - linked_count = 0 - error_count = 0 - - for hgvs_string, response in zip(hgvs_list, registered_alleles): - mapped_variant_ids = variant_post_mapped_hgvs[hgvs_string] - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(mapped_variant_ids)) - ).all() - - if "errorType" in response: - error_count += 1 - logger.warning( - msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + + # Bulk-load every allele that could be linked in one query, keyed by id, rather than + # issuing a SELECT per CAID inside the response loop. + alleles_by_id = { + allele.id: allele + for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() + } + + # Reverse translation can push allele counts high enough that a single PUT is untenable, so + # dispatch in fixed-size chunks. Each batch is reconciled independently: a transient failure + # (or a broken one-result-per-input contract) fails only that batch's alleles, not the run. + batches = list(batched(list(hgvs_to_allele_ids.keys()), DEFAULT_CAR_SUBMISSION_BATCH_SIZE)) + job_manager.save_to_context( + { + "car_submission_batch_size": DEFAULT_CAR_SUBMISSION_BATCH_SIZE, + "car_submission_batch_count": len(batches), + } + ) + + for batch_idx, hgvs_list in enumerate(batches): + # Spread the 15→60 window across batches so progress advances as each chunk is dispatched. + job_manager.update_progress( + 15 + (45 * batch_idx) // len(batches), + 100, + f"Submitting alleles to CAR (batch {batch_idx + 1} / {len(batches)}).", + ) + registered_alleles = car_service.dispatch_submissions(hgvs_list) + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail this batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing this batch." + ), extra=job_manager.logging_context(), ) - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": { + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] + + if "errorType" in response: + logger.warning( + msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ "submitted_hgvs": hgvs_string, "car_error_type": response.get("errorType"), "car_error_message": response.get("message"), }, - }, - current=True, - ) + ) - else: - linked_count += 1 - caid = response["@id"].split("/")[-1] - for mapped_variant in mapped_variants: - mapped_variant.clingen_allele_id = caid - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"clingen_allele_id": caid}}, - current=True, - ) + continue - # Any HGVS strings CAR did not respond to (network drop, service-side omission). - # Use EXTERNAL_SERVICE_REJECTED for explicit CAR errors, EXTERNAL_API_ERROR for silent failures. - no_response_hgvs = hgvs_list[len(registered_alleles) :] - for hgvs_string in no_response_hgvs: - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(variant_post_mapped_hgvs[hgvs_string])) - ).all() - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": {"submitted_hgvs": hgvs_string}, - }, - current=True, - ) + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: + logger.error( + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) + + continue + + caid = caid_iri.split("/")[-1] + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant + # violation. Do not overwrite; record a failure with full audit context. + if prior_caid and prior_caid != caid: + logger.error( + msg=( + f"CAR returned a different CAID for allele {allele_id}: " + f"stored={prior_caid!r}, returned={caid!r}. " + "Not overwriting. Investigate immediately." + ), + extra=job_manager.logging_context(), + ) + + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.CAID_CONFLICT, + error_message="CAR returned a CAID that conflicts with the stored value.", + metadata={ + "clingen_allele_id": prior_caid, + "conflicting_caid": caid, + "submitted_hgvs": hgvs_string, + }, + ) + + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + metadata={"clingen_allele_id": caid}, + ) + + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, + ) annotation_manager.flush() - failed_count = no_hgvs_count + error_count + len(no_response_hgvs) + outcome_data = _outcome_data() - # When all registrations fail we will not be able to render any annotations. Fail the job - # to explicitly halt the pipeline. - if linked_count == 0: - error_message = f"CAR submission failed for all {len(hgvs_list)} variants in score set {score_set.urn}." + # When no allele ended up with a CAID (none linked this run, none already registered), the + # pipeline cannot continue — downstream jobs need CAIDs to function. + if not linked_allele_ids and not preexisting_allele_ids: + error_message = ( + f"CAR submission failed for all {outcome_data['submitted_allele_count']} " + f"submitted alleles in score set {score_set.urn}." + ) logger.error(msg=error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": len(hgvs_list), "matched_count": 0, "failed_count": failed_count}, + data=outcome_data, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) - if failed_count > 0: - # CAR rejections are typically per-variant data quality issues (e.g. invalid HGVS) rather than - # systemic failures. Per-variant AnnotationStatus.FAILED records are already written above for - # traceability. We continue the pipeline so that successfully registered variants still receive - # downstream annotations (warm_clingen_cache, gnomAD, ClinVar, HGVS, translations). + if outcome_data["failed_allele_count"] > 0: logger.warning( - msg=f"CAR submission failed for {failed_count} of {len(hgvs_list)} variants in score set {score_set.urn}.", + msg=f"CAR submission failed for {outcome_data['failed_allele_count']} alleles in score set {score_set.urn}.", extra=job_manager.logging_context(), ) logger.info(msg="Completed CAR mapped resource submission", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(hgvs_list), "matched_count": linked_count, "failed_count": failed_count} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) @with_pipeline_management @@ -303,7 +485,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: - Submits data to ClinGen Linked Data Hub Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-variant submitted/failed counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -331,15 +513,26 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT) ldh_service.authenticate() - # Fetch mapped variants with post-mapped data for the score set - variant_objects = job_manager.db.execute( - select(Variant, MappedVariant) - .join(MappedVariant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() + # Fetch each variant's authoritative allele for the score set. Post-mapped data and HGVS + # come from the Allele; pre-mapped data and the mapping API version come from the + # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each + # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, + # which registers a CAID per allele). + variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( + job_manager.db.execute( + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) + ) + .tuples() + .all() + ) # Track total variants to submit job_manager.save_to_context({"total_variants_to_submit_ldh": len(variant_objects)}) @@ -354,20 +547,22 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.update_progress(10, 100, f"Submitting {len(variant_objects)} mapped variants to LDH.") # Build submission content - variant_content = [] - variant_for_urn = {} - for variant, mapped_variant in variant_objects: - variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) + variant_content: list[tuple[str, Variant, MappingRecord, AlleleModel]] = [] + variant_for_urn: dict[str, Variant] = {} + for variant, mapping_record, allele in variant_objects: + # cis-phased blocks are skipped here pending ClinGen guidance TODO#764 + variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant.id}. Skipping submission of this variant.", + msg=f"Could not construct a valid HGVS string for allele {allele.id} (variant {variant.urn}). Skipping submission of this variant.", extra=job_manager.logging_context(), ) continue - variant_content.append((variation, variant, mapped_variant)) - variant_for_urn[variant.urn] = variant + variant_content.append((variation, variant, mapping_record, allele)) + # TODO#372: nullable URNs + variant_for_urn[variant.urn] = variant # type: ignore if not variant_content: logger.warning( @@ -394,8 +589,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: } ) - # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) submitted_variant_urns = set() for success in submission_successes: logger.debug( @@ -404,17 +601,22 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) submitted_urn = success["data"]["entId"] - submitted_variant = variant_for_urn[submitted_urn] + submitted_variant = variant_for_urn.get(submitted_urn) + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. + if submitted_variant is None: + logger.warning( + msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", + extra=job_manager.logging_context(), + ) + continue - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=submitted_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": {"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, - }, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + metadata={"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, ) submitted_variant_urns.add(submitted_urn) @@ -422,7 +624,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # especially when submission occurred in batch. Save all failures generically here. # Note that failures may not be present in the submission failures list, but they are # guaranteed to be absent from the successes list. - for failure_urn in set(variant_for_urn.keys()) - submitted_variant_urns: + failed_variant_urns = set(variant_for_urn.keys()) - submitted_variant_urns + for failure_urn in failed_variant_urns: logger.error( msg=f"Failed to submit mapped variant to LDH: {failure_urn}", extra=job_manager.logging_context(), @@ -430,23 +633,25 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: failed_variant = variant_for_urn[failure_urn] - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=failed_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to submit variant to ClinGen Linked Data Hub.", - }, - current=True, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "Failed to submit variant to ClinGen Linked Data Hub."}, ) annotation_manager.flush() + # Report per-variant counts (matching the annotations written above), not the per-batch + # counts returned by the service — the two use different denominators. + submitted_count = len(submitted_variant_urns) + failed_count = len(failed_variant_urns) + if submission_failures: logger.warning( - msg=f"LDH mapped resource submission encountered {len(submission_failures)} failures.", + msg=f"LDH mapped resource submission encountered {len(submission_failures)} batch failures " + f"({failed_count} variants unconfirmed).", extra=job_manager.logging_context(), ) @@ -460,7 +665,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": 0, "failed_count": len(submission_failures)}, + data={"submitted_count": submitted_count, "failed_count": failed_count}, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -470,6 +675,4 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(submission_successes), "failed_count": len(submission_failures)} - ) + return JobExecutionOutcome.succeeded(data={"submitted_count": submitted_count, "failed_count": failed_count}) diff --git a/src/mavedb/worker/jobs/external_services/clingen_cache.py b/src/mavedb/worker/jobs/external_services/clingen_cache.py index 10890f23f..d530d881f 100644 --- a/src/mavedb/worker/jobs/external_services/clingen_cache.py +++ b/src/mavedb/worker/jobs/external_services/clingen_cache.py @@ -15,11 +15,10 @@ from sqlalchemy import select from mavedb.lib.clingen.allele_registry import get_clingen_allele_data +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import CLINGEN_CACHE_WARMING_CONCURRENCY from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -55,19 +54,16 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> job_manager.update_progress(0, 100, "Starting ClinGen cache pre-warming.") logger.info("Starting ClinGen cache pre-warming", extra=job_manager.logging_context()) - # Get distinct clingen_allele_ids for this score set's current mapped variants - allele_ids = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.isnot(None), - # Exclude multi-variant IDs (comma-separated) — they can't be fetched individually - MappedVariant.clingen_allele_id.not_like("%,%"), - ) - .distinct() - ).all() + # Get distinct clingen_allele_ids registered for alleles in this score set. + # Exclude None and comma-separated multi-variant IDs that can't be fetched individually. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_ids = list( + { + row.clingen_allele_id + for row in allele_rows + if row.clingen_allele_id is not None and "," not in row.clingen_allele_id + } + ) total = len(allele_ids) job_manager.save_to_context({"total_allele_ids_to_warm": total}) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 68a34a04b..d2482c6db 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -1,19 +1,18 @@ -"""ClinVar integration jobs for variant annotation +"""ClinVar integration jobs for variant annotation. -This module contains job definitions and utility functions for integrating ClinVar -variant data into MaveDB. It includes functions to fetch and parse ClinVar variant -summary data, and update MaveDB records with the latest ClinVar annotations. +Links deduplicated alleles to ClinVar clinical-control data across every archival ClinVar +release. Each release is a distinct, versioned ``ClinvarControl`` entity, and an allele +accumulates one live ``ClinvarAlleleLink`` per release it appears in (multi-live, unlike +gnomAD/VEP which hold one live result per allele). Both ClinGen API calls and ClinVar TSV data fetches are automatically cached using aiocache with Redis backend: - ClinGen API calls: 24-hour TTL - ClinVar TSV files: 90-day TTL (archival data doesn't change) - -This significantly reduces redundant network requests when refreshing ClinVar -controls across multiple months/years. """ import logging +from collections import Counter from datetime import datetime import requests @@ -22,14 +21,17 @@ from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.allele_registry import get_associated_clinvar_allele_id +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -42,28 +44,71 @@ CLINVAR_START_MONTH = 2 -def generate_clinvar_versions() -> list[tuple[int, int]]: +def _generate_clinvar_versions() -> list[tuple[int, int]]: """Generate all ClinVar version (year, month) pairs from Feb 2015 to current Jan. Returns a list of (year, month) tuples representing each ClinVar archival snapshot that should be processed. """ current_year = datetime.now().year - versions = [(CLINVAR_START_YEAR, CLINVAR_START_MONTH)] - for year in range(CLINVAR_START_YEAR + 1, current_year + 1): - versions.append((year, 1)) - return versions + first_version = (CLINVAR_START_YEAR, CLINVAR_START_MONTH) + archival_versions = [(year, 1) for year in range(CLINVAR_START_YEAR + 1, current_year + 1)] + return [first_version, *archival_versions] + + +def _annotate_clinvar( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + source_version: str, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CLINVAR_CONTROL event for an allele at a given ClinVar release. + + The single choke point for ClinVar's status writes. One event per (allele, release); provenance + (which variants drove the linkage) is derived from the live links as-of the event, not fanned out + here. Unlike gnomAD/VEP, ClinVar is multi-version, so ``source_version`` (the ClinVar release) + distinguishes an allele's events across releases. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINVAR_CONTROL, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) @with_pipeline_management async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Refresh ClinVar clinical control data across all archival versions. - - Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each - subsequent year through the current year), fetching TSV data and updating - clinical control records for all mapped variants in the score set. Individual - version failures are logged and skipped — the job continues processing - remaining versions. + """Link deduplicated alleles to ClinVar clinical-control data across all archival versions. + + Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each subsequent year through + the current year). For each version it resolves each allele's ClinGen Allele ID (CAID) to a ClinVar + allele id, upserts the versioned :class:`ClinvarControl`, and establishes a live + :class:`ClinvarAlleleLink`. An allele accumulates one live link per release (multi-live). Individual + version fetch failures are logged and skipped — the job continues with the remaining versions. + + Job Parameters: + - score_set_id (int): The ID of the ScoreSet whose alleles to process. + - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the per-version skip and re-resolve every allele. The link + write still get-or-creates (no duplicate live link), so a forced re-run of unchanged data + writes no new links. + + Side Effects: + - Creates ClinvarControl and ClinvarAlleleLink rows. + + Returns: + JobExecutionOutcome: outcome with version and per-link counts. """ job = job_manager.get_job() @@ -72,8 +117,9 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] - versions = generate_clinvar_versions() + versions = _generate_clinvar_versions() job_manager.save_to_context( { @@ -83,26 +129,56 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag "correlation_id": correlation_id, "versions": versions, "total_versions": len(versions), + "force": force, } ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - variants_to_refresh = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys + # on is often the RT-derived one. Events are allele-keyed, so every allele is recorded per release + # (no fan-out). + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} + job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + + # Link counts accumulate across all versions (an allele may link in every release it appears in). + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) + + if not allele_data: + logger.warning( + msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded( + data={"versions_completed": 0, "versions_total": len(versions), **dict(annotation_counts)} ) - ).all() - total_variants_to_refresh = len(variants_to_refresh) - job_manager.save_to_context({"total_variants_to_refresh": total_variants_to_refresh}) - total_refreshed = 0 - total_failed = 0 - versions_completed = 0 + all_allele_ids = set(allele_data.keys()) + + def alleles_linked_at_version(clinvar_version: str) -> set[int]: + """Allele ids (within the work set) holding a live ClinvarAlleleLink to a control of this version.""" + return set( + job_manager.db.scalars( + select(ClinvarAlleleLink.allele_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(all_allele_ids)) + .where(ClinvarAlleleLink.current) + .where(ClinvarControl.db_version == clinvar_version) + ).all() + ) + versions_completed = 0 for version_index, (year, month) in enumerate(versions): clinvar_version = f"{month:02d}_{year}" job_manager.save_to_context({"current_version": clinvar_version, "version_index": version_index}) @@ -125,163 +201,204 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) continue - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in variants_to_refresh: - clingen_id = mapped_variant.clingen_allele_id - - if clingen_id is None: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Mapped variant does not have an associated ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + # Cost: skip alleles already linked at this version (an archival release cannot change). force + # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. + already_linked = set() if force else alleles_linked_at_version(clinvar_version) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): + # ClinVar is a nucleotide-level database, skip any protein level alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + source_version=clinvar_version, + error_message="Protein-level alleles are not linked to ClinVar; see the nucleotide encodings.", + metadata={"clingen_allele_id": caid}, ) continue - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", - }, - current=True, - replace_all_versions=False, + if allele_id in already_linked: + annotation_counts["preexisting_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid}, + ) + continue + + # A cis-block (multi-variant) CAID structurally cannot key ClinVar — a terminal gap, not + # a statement about ClinVar's contents. + if "," in caid: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.MULTI_VARIANT_CAID, + source_version=clinvar_version, + error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", + metadata={"clingen_allele_id": caid}, ) continue try: - clinvar_allele_id = await get_associated_clinvar_allele_id(clingen_id) # type: ignore + clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", - }, - current=True, - replace_all_versions=False, + annotation_counts["failed_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=clinvar_version, + error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", + metadata={"clingen_allele_id": caid}, ) logger.error( - f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {clingen_id}.", + f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {caid}.", extra=job_manager.logging_context(), exc_info=exc, ) - total_failed += 1 continue + # ClinGen has no ClinVar AlleleID for this CAID — the allele is not a ClinVar control: an + # informative negative about the source, not a pipeline gap. if not clinvar_allele_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": "No ClinVar allele ID found for ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, + error_message="No ClinVar allele ID found for ClinGen allele ID.", + metadata={"clingen_allele_id": caid}, ) continue + # The allele has a ClinVar AlleleID but it is absent from this release's snapshot — a + # genuine, version-scoped negative (ClinVar queried, nothing for this release). if clinvar_allele_id not in tsv_data: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No ClinVar data found for ClinVar allele ID.", - }, - current=True, - replace_all_versions=False, + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, + error_message="No ClinVar data found for ClinVar allele ID.", + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) continue variant_data = tsv_data[clinvar_allele_id] - identifier = str(clinvar_allele_id) - # Atomic upsert — avoids a check-then-act race when two - # refresh_clinvar_controls jobs run concurrently for different - # score sets and encounter the same (db_name, db_identifier, - # db_version) tuple. ON CONFLICT DO UPDATE is guaranteed to - # return exactly one row regardless of concurrent inserts. + # Atomic upsert — avoids a check-then-act race when two refresh_clinvar_controls jobs run + # concurrently for different score sets and encounter the same + # (db_name, db_identifier, db_version) tuple. ON CONFLICT DO UPDATE returns exactly one row. upsert_stmt = ( - pg_insert(ClinicalControl) + pg_insert(ClinvarControl) .values( - db_identifier=identifier, + db_identifier=str(clinvar_allele_id), db_version=clinvar_version, db_name="ClinVar", gene_symbol=variant_data.get("GeneSymbol"), clinical_significance=variant_data.get("ClinicalSignificance"), clinical_review_status=variant_data.get("ReviewStatus"), + clinvar_variation_id=variant_data.get("VariationID"), ) .on_conflict_do_update( - constraint="uq_clinical_controls_db_name_identifier_version", + constraint="uq_clinvar_controls_db_name_identifier_version", set_={ "gene_symbol": variant_data.get("GeneSymbol"), "clinical_significance": variant_data.get("ClinicalSignificance"), "clinical_review_status": variant_data.get("ReviewStatus"), + "clinvar_variation_id": variant_data.get("VariationID"), }, ) - .returning(ClinicalControl) + .returning(ClinvarControl) ) - clinvar_variant = job_manager.db.scalars(upsert_stmt).one() - - job_manager.db.add(clinvar_variant) - job_manager.db.flush() - - if clinvar_variant not in mapped_variant.clinical_controls: - mapped_variant.clinical_controls.append(clinvar_variant) - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "clinvar_allele_id": clinvar_allele_id, - }, - }, - current=True, - replace_all_versions=False, + clinvar_control = job_manager.db.scalars(upsert_stmt).one() + + # At most one live link per (allele, release). Normally a release is immutable, so the + # allele's live link for this version is either a reconfirm of this same control (no-op) or + # absent (insert) — multi-live accumulates only across *different* releases, never supersedes. + # Defensive guard: if the allele already holds a live link to a *different* control of this + # same version, the release re-resolved under us (re-ingestion / upstream correction — should + # never happen for archival data). Supersede newest-wins with a shared timestamp and log. + live_link = job_manager.db.scalar( + select(ClinvarAlleleLink) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.current, + ClinvarControl.db_version == clinvar_version, + ) ) + if live_link is None: + job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) + annotation_counts["created_link_count"] += 1 + reason = EventReason.CREATED + elif live_link.clinvar_control_id == clinvar_control.id: + annotation_counts["preexisting_link_count"] += 1 + reason = EventReason.PREEXISTING + else: + live_link.supersede_with( + job_manager.db, + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id), + ) + annotation_counts["created_link_count"] += 1 + reason = EventReason.SUPERSEDED + logger.warning( + msg=( + f"Allele {allele_id} held a live ClinVar link to control " + f"{live_link.clinvar_control_id} for version {clinvar_version}, but re-resolved to " + f"control {clinvar_control.id} (ClinVar allele {clinvar_allele_id}). Superseding " + "newest-wins; archival ClinVar data should be immutable — investigate the upstream " + "re-resolution." + ), + extra=job_manager.logging_context(), + ) - total_refreshed += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, + ) annotation_manager.flush() + job_manager.db.flush() versions_completed += 1 logger.info( - f"Completed ClinVar version {clinvar_version} for {total_variants_to_refresh} variants.", + f"Completed ClinVar version {clinvar_version} for {len(allele_data)} alleles.", extra=job_manager.logging_context(), ) logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{total_refreshed} variant-version annotations.", + f"{annotation_counts['created_link_count']} new links, " + f"{annotation_counts['preexisting_link_count']} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and total_refreshed == 0: - error_message = ( - f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - ) + if ( + annotation_counts["failed_link_count"] > 0 + and annotation_counts["created_link_count"] == 0 + and annotation_counts["preexisting_link_count"] == 0 + ): + error_message = f"All {annotation_counts['failed_link_count']} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." logger.error(error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -289,16 +406,12 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": 0, + **dict(annotation_counts), }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": versions_completed, - "versions_total": len(versions), - "variant_annotations": total_refreshed, - } + data={"versions_completed": versions_completed, "versions_total": len(versions), **dict(annotation_counts)} ) diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 969c3a20e..4c688692d 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -1,29 +1,33 @@ """gnomAD variant linking jobs for population frequency annotation. -This module handles linking of mapped variants to gnomAD (Genome Aggregation Database) +This module handles linking of deduplicated alleles to gnomAD (Genome Aggregation Database) variants to provide population frequency and other genomic context information. This enrichment helps researchers understand the clinical significance and rarity of variants in their datasets. """ import logging -from typing import Sequence +from collections import Counter from sqlalchemy import select from mavedb.db import athena from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, + GnomadLinkVerdict, gnomad_variant_data_for_caids, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -31,17 +35,48 @@ logger = logging.getLogger(__name__) +def _annotate_gnomad( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one GNOMAD_ALLELE_FREQUENCY event for an allele (frequency is an allele-level fact). + + The single choke point for gnomAD's status writes. One event per allele, stamped at the current + gnomAD version; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=GNOMAD_DATA_VERSION, + metadata=meta or None, + ) + + @with_pipeline_management async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ - Link mapped variants to gnomAD variants based on ClinGen Allele IDs (CAIDs). - This job fetches mapped variants associated with a given score set that have CAIDs, - retrieves corresponding gnomAD variant data, and establishes links between them - in the database. + Link deduplicated alleles to gnomAD variants based on ClinGen Allele IDs (CAIDs). + This job fetches the current authoritative alleles of a score set that carry CAIDs, + retrieves corresponding gnomAD variant data, and establishes valid-time links between them. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants to process. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the version-keyed skip and re-fetch every CAID-bearing + allele. The linker still supersedes only on change, so a forced re-run of unchanged data + writes no new links. Use for re-ingestion or to heal suspected link corruption. Args: ctx (dict): The job context dictionary. @@ -49,10 +84,10 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager (JobManager): The job manager instance for database and logging operations. Side Effects: - - Updates MappedVariant records to link to gnomAD variants. + - Creates GnomadAlleleLink rows linking alleles to gnomAD variants. Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -63,6 +98,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] # Setup initial context and progress job_manager.save_to_context( @@ -76,87 +112,144 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # We filter out mapped variants that do not have a CAID, so this query is typed # as a Sequence[str]. Ignore MyPy's type checking here. - variant_caids: Sequence[str] = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), - ) - ).all() # type: ignore + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows + # is often the RT-derived one. Events are allele-keyed, so every linked allele is recorded. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} + + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "skipped_allele_count": 0, + } + ) - num_variant_caids = len(variant_caids) - job_manager.save_to_context({"num_variants_to_link_gnomad": num_variant_caids}) + num_alleles_with_caids = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) - if not variant_caids: + if not allele_data: logger.warning( - msg="No current mapped variants with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", + msg="No current alleles with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"linked_count": 0, "skipped_count": 0}) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) - job_manager.update_progress(10, 100, f"Found {num_variant_caids} variants with CAIDs to link to gnomAD variants.") - logger.info( - msg="Found current mapped variants with CAIDs for this score set. Attempting to link them to gnomAD variants.", - extra=job_manager.logging_context(), + job_manager.update_progress( + 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." ) - # Fetch gnomAD variant data for the CAIDs - with athena.engine.connect() as athena_session: - logger.debug("Fetching gnomAD variants from Athena.") - gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) + def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live gnomAD link at the current gnomAD version.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(GnomadAlleleLink.allele_id) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.current) + .where(GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ).all() + ) - num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + # Skip alleles already linked at the current version (they can't change). force re-fetches + # all; the linker still supersedes only on change, so a forced no-op writes nothing. + already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) + # Never query Athena for protein alleles — gnomAD is nucleotide-level, so they can't match. They + # are still recorded as not-applicable in the loop below; this only spares them the round-trip. + variant_caids = sorted( + { + allele_data[aid] + for aid in allele_data + if aid not in already_current and allele_levels.get(aid) != SequenceLevel.protein.value + } + ) - # NOTE: Proceed intentionally with linking even if no matches were found, to record skipped annotations. + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_caids_to_query": len(variant_caids), + "force": force, + } + ) - job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) - job_manager.update_progress(75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs.") + verdicts: dict[int, GnomadLinkVerdict] = {} + if variant_caids: + with athena.engine.connect() as athena_session: + logger.debug("Fetching gnomAD variants from Athena.") + gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - # Link mapped variants to gnomAD variants - logger.info(msg="Attempting to link mapped variants to gnomAD variants.", extra=job_manager.logging_context()) - num_linked_gnomad_variants = link_gnomad_variants_to_mapped_variants(job_manager.db, gnomad_variant_data) - job_manager.db.flush() + num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) + job_manager.update_progress( + 75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs." + ) - # For variants which are not linked, create annotation status records indicating skipped linkage - mapped_variants_with_caids = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), + logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) + verdicts = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + job_manager.db.flush() + else: + logger.info( + msg="All CAID-bearing alleles are already linked at the current gnomAD version; skipping Athena query.", + extra=job_manager.logging_context(), ) - ).all() - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in mapped_variants_with_caids: - if not mapped_variant.gnomad_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No gnomAD variant could be linked for this mapped variant.", - }, - current=True, + job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): + # gnomAD is a nucleotide-level, genomic-coordinate resource. Skip any protein alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + error_message="Protein-level alleles are not linked to gnomAD; see the nucleotide encodings.", + metadata={"clingen_allele_id": caid}, + ) + continue + + verdict = verdicts.get(allele_id) + if verdict is GnomadLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + metadata={"clingen_allele_id": caid}, + ) + elif allele_id in already_current or verdict is GnomadLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": caid}, + ) + else: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + error_message="No gnomAD variant could be linked for this allele.", + metadata={"clingen_allele_id": caid}, ) annotation_manager.flush() - # Save final context and progress - job_manager.save_to_context({"num_mapped_variants_linked_to_gnomad_variants": num_linked_gnomad_variants}) - logger.info(msg="Done linking gnomAD variants to mapped variants.", extra=job_manager.logging_context()) + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) + logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "linked_count": num_linked_gnomad_variants, - "skipped_count": num_variant_caids - num_linked_gnomad_variants, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/jobs/external_services/hgvs.py b/src/mavedb/worker/jobs/external_services/hgvs.py deleted file mode 100644 index 0b4687398..000000000 --- a/src/mavedb/worker/jobs/external_services/hgvs.py +++ /dev/null @@ -1,298 +0,0 @@ -"""ClinGen allele HGVS population jobs for mapped variant annotation. - -This module populates mapped variants with HGVS representations (genomic, coding, -protein) by querying the ClinGen Allele Registry. It uses ClinGen allele IDs -(CAIDs) already associated with mapped variants to look up standardized HGVS -nomenclature at different levels (hgvs_g, hgvs_c, hgvs_p), plus the assay-level -HGVS derived from post-mapped VRS data. -""" - -import logging -from typing import Optional - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - extract_hgvs_from_ca_allele_data, - extract_hgvs_from_pa_allele_data, - get_clingen_allele_data, -) -from mavedb.lib.slack import log_and_send_slack_message -from mavedb.lib.target_genes import get_target_coding_info -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_hgvs_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate mapped variants with HGVS representations for a score set. - - Queries the ClinGen Allele Registry using existing ClinGen allele IDs to populate - standardized HGVS nomenclature (genomic, coding, protein) on mapped variants. - Also extracts the assay-level HGVS from post-mapped VRS data. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - - Args: - ctx: Worker context containing DB and Redis connections. - job_id: The ID of the job run. - job_manager: Manager for job lifecycle and DB operations. - - Side Effects: - - Updates MappedVariant records with hgvs_assay_level, hgvs_g, hgvs_c, hgvs_p. - - Creates AnnotationStatus records for each processed variant. - - Returns: - JobExecutionOutcome indicating success, failure, or skip. - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_hgvs_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting mapped HGVS population.") - logger.info(msg="Started mapped HGVS population", extra=job_manager.logging_context()) - - # Determine target info; multi-target score sets are not yet supported - try: - target_is_coding, transcript_accession = get_target_coding_info(score_set) - except NotImplementedError: - logger.warning( - msg="Multi-target score sets not supported for HGVS population. Skipping.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.skipped(data={"reason": "Multi-target score sets not supported"}) - - job_manager.save_to_context({"target_is_coding": target_is_coding, "transcript_accession": transcript_accession}) - logger.info( - msg=f"Target info resolved: coding={target_is_coding}, transcript={transcript_accession}", - extra=job_manager.logging_context(), - ) - - # Fetch current mapped variants for the score set - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - total_variants = len(variant_rows) - job_manager.save_to_context({"total_variants": total_variants}) - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set. Skipping HGVS population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"populated_count": 0, "skipped_count": 0, "failed_count": 0}) - - job_manager.update_progress(5, 100, f"Processing {total_variants} mapped variants for HGVS population.") - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - populated_count = 0 - skipped_count = 0 - failed_count = 0 - - for index, (variant_id, mapped_variant) in enumerate(variant_rows): - # Periodic progress updates - if total_variants > 0 and index % max(total_variants // 20, 1) == 0: - progress = 5 + int((index / total_variants) * 90) - job_manager.update_progress(progress, 100, f"Processing HGVS for variant {index + 1}/{total_variants}.") - logger.info( - "Processing variant %s/%s: variant_id=%s", - index + 1, - total_variants, - variant_id, - extra=job_manager.logging_context(), - ) - - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - clingen_id = mapped_variant.clingen_allele_id - - job_manager.save_to_context( - { - "mapped_variant_id": mapped_variant.id, - "clingen_allele_id": clingen_id, - "progress_index": index, - } - ) - - if not clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "No ClinGen allele ID available for ClinGen HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: no ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Skip multi-variant allele IDs (comma-separated) - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs not supported for HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: multi-variant ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Query ClinGen API for allele data - try: - allele_data = await get_clingen_allele_data(clingen_id) - except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to fetch ClinGen allele data: {str(exc)}", - }, - current=True, - ) - logger.error( - "ClinGen API request failed for allele %s.", - clingen_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed_count += 1 - continue - - if allele_data is None: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": f"ClinGen allele {clingen_id} not found in the registry.", - }, - current=True, - ) - logger.debug( - "ClinGen allele %s not found in registry. Skipping variant %s.", - clingen_id, - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Extract HGVS based on allele type - if clingen_id.startswith("CA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_ca_allele_data( - allele_data, target_is_coding, transcript_accession - ) - elif clingen_id.startswith("PA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_pa_allele_data(allele_data) - - # Update mapped variant - mapped_variant.hgvs_g = hgvs_g - mapped_variant.hgvs_c = hgvs_c - mapped_variant.hgvs_p = hgvs_p - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "hgvs_g": hgvs_g, - "hgvs_c": hgvs_c, - "hgvs_p": hgvs_p, - }, - }, - current=True, - ) - populated_count += 1 - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) - logger.info( - msg=f"Completed mapped HGVS population: {populated_count} populated, {skipped_count} skipped, {failed_count} failed.", - extra=job_manager.logging_context(), - ) - - if failed_count > 0 and populated_count == 0: - log_and_send_slack_message( - f"All {failed_count} variants failed HGVS population for score set {score_set.urn}. Possible ClinGen API outage.", - job_manager.logging_context(), - logging.ERROR, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/variant_translation.py b/src/mavedb/worker/jobs/external_services/variant_translation.py deleted file mode 100644 index ddec4b731..000000000 --- a/src/mavedb/worker/jobs/external_services/variant_translation.py +++ /dev/null @@ -1,371 +0,0 @@ -"""ClinGen allele variant translation jobs for mapping PA<->CA allele relationships. - -This module populates the variant_translations table with relationships between -protein allele (PA) and nucleotide allele (CA) ClinGen IDs. For CA alleles, it -looks up MANE canonical PA IDs and their matching registered transcript CA IDs. -For PA alleles, it looks up matching registered transcript CA IDs directly. -""" - -import logging - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - expand_allele_ids, - get_canonical_pa_ids, - get_matching_registered_ca_ids, -) -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.lib.variant_translations import upsert_variant_translations -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_variant_translations_for_score_set( - ctx: dict, job_id: int, job_manager: JobManager -) -> JobExecutionOutcome: - """Populate variant translations (PA<->CA relationships) for a score set. - - Queries the ClinGen Allele Registry to discover relationships between protein - allele (PA) and nucleotide allele (CA) ClinGen IDs, then stores them in the - variant_translations table. Each unique allele ID is processed once even if - shared across multiple mapped variants. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_variant_translations_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting variant translation population.") - logger.info(msg="Started variant translation population.", extra=job_manager.logging_context()) - - # Fetch all current mapped variants with their ClinGen allele IDs - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant.clingen_allele_id) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .join(ScoreSet, Variant.score_set_id == ScoreSet.id) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - # Deduplicate: multiple mapped variants can share the same allele ID, but we only - # need to query the ClinGen API once per unique ID. Track which variants map to each - # allele so we can record annotations for all of them after a single lookup. - allele_to_variants: dict[str, list[int]] = {} - for variant_id, clingen_allele_id in variant_rows: - if not clingen_allele_id: - continue - - for individual_id in expand_allele_ids([clingen_allele_id]): - allele_to_variants.setdefault(individual_id, []).append(variant_id) - - unique_allele_ids = list(allele_to_variants.keys()) - total_alleles = len(unique_allele_ids) - job_manager.save_to_context({"total_variants": len(variant_rows), "unique_allele_ids": total_alleles}) - - if not unique_allele_ids: - logger.warning( - msg="No ClinGen allele IDs found on mapped variants.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - job_manager.update_progress(5, 100, f"Processing {total_alleles} unique allele IDs for variant translations.") - logger.info( - "Processing %s unique allele IDs for variant translations.", - total_alleles, - extra=job_manager.logging_context(), - ) - - total_created = 0 - total_skipped = 0 - total_failed = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - for index, allele_id in enumerate(unique_allele_ids): - if total_alleles > 0 and index % max(total_alleles // 20, 1) == 0: - progress = 5 + int((index / total_alleles) * 90) - job_manager.update_progress(progress, 100, f"Processing allele {index + 1}/{total_alleles}.") - logger.info( - "Processing allele %s/%s: %s", - index + 1, - total_alleles, - allele_id, - extra=job_manager.logging_context(), - ) - - job_manager.save_to_context( - { - "current_allele_id": allele_id, - "progress_index": index, - } - ) - - variant_ids = allele_to_variants[allele_id] - - if allele_id.startswith("CA"): - # CA (nucleotide) alleles: look up the MANE canonical protein alleles (PAs) for - # this CA, then for each PA discover all registered transcript-level CAs. This - # CA -> PA -> CA expansion builds the full translation graph so we can link - # nucleotide variants to their protein equivalents and vice versa. - try: - canonical_pa_ids = await get_canonical_pa_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for canonical PA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error looking up PA IDs for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not canonical_pa_ids: - # Noncoding variants won't have protein alleles — this is expected and not an error. - logger.debug( - "No canonical PA IDs found for %s (may be noncoding).", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No canonical PA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - created = 0 - failed = 0 - translation_pairs: set[tuple[str, str]] = set() - for pa_id in canonical_pa_ids: - # Record the direct PA <-> original CA relationship. - translation_pairs.add((pa_id, allele_id)) - - # Then expand: find all other CAs registered under this PA so we capture - # alternate transcript-level representations of the same protein change. - try: - ca_ids = await get_matching_registered_ca_ids(pa_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - pa_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed += 1 - continue - - for ca_id in ca_ids: - translation_pairs.add((pa_id, ca_id)) - - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED if failed > 0 else AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - "pa_lookups_failed": failed, - "pa_lookups_total": len(canonical_pa_ids), - }, - }, - current=True, - ) - - total_created += created - total_failed += failed - - elif allele_id.startswith("PA"): - # PA (protein) alleles: directly look up all registered transcript-level CAs. - # This is simpler than the CA path since we already have the protein allele. - try: - ca_ids = await get_matching_registered_ca_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not ca_ids: - logger.warning( - "No matching registered transcript CA IDs for PA allele %s. This is unexpected.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No registered transcript CA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - translation_pairs = set([(allele_id, ca_id) for ca_id in ca_ids]) - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - }, - }, - current=True, - ) - - total_created += created - - else: - logger.warning( - "Unrecognized ClinGen allele ID format: %s. Skipping.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": f"Unrecognized allele ID format: {allele_id}", - }, - current=True, - ) - total_skipped += len(variant_ids) - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) - logger.info( - "Completed variant translation population: %s created, %s skipped, %s failed.", - total_created, - total_skipped, - total_failed, - extra=job_manager.logging_context(), - ) - - if total_failed > 0 and total_created == 0: - error_message = f"All {total_failed} variant translation lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - logger.error(error_message, extra=job_manager.logging_context()) - job_manager.db.flush() - return JobExecutionOutcome.failed( - reason=error_message, - data={ - "translations_created": 0, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - }, - failure_category=FailureCategory.DEPENDENCY_FAILURE, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index ba577f230..0264938f4 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -1,28 +1,37 @@ """VEP functional consequence jobs for variant effect prediction. -This module handles the submission and processing of variant effect predictions -using the Ensembl VEP API. - -The processing is asynchronous, requiring batch submission of HGVS strings -to the VEP API with fallback to Variant Recoder when necessary. +This module links deduplicated alleles to their Ensembl VEP functional consequence. Submission is +batched against the VEP API, with a Variant Recoder fallback for HGVS strings VEP cannot resolve +directly (notably protein HGVS). """ import asyncio import logging import os +from collections import Counter from datetime import date +import requests from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched -from mavedb.lib.vep import VEP_CONSEQUENCES, get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + VEP_CONSEQUENCES, + VepLinkVerdict, + VepResolution, + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -34,17 +43,198 @@ _RECODER_CONCURRENCY = int(os.getenv("RECODER_CONCURRENCY", "5")) +def _annotate_vep( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + source_version: str, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one VEP_FUNCTIONAL_CONSEQUENCE event for an allele (the consequence is an allele-level fact). + + The single choke point for VEP's status writes. One event per allele, stamped with the Ensembl + release queried; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. The consequence value itself is not embedded — it lives in + the ``VepAlleleConsequence`` value table, joinable by ``allele_id`` + ``source_version``. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) + + +def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: + """Build VEP's HGVS input for an allele: each allele will only have one of these, so the first + non-null is the one to submit. + """ + return row.hgvs_g or row.hgvs_c or row.hgvs_p or None + + +async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> VepResolution: + """Resolve a set of HGVS strings to their most-severe VEP consequence, splitting empty from errored. + + Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry + with a null consequence is treated as a miss — VEP knew the variant but could not classify it). + Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to + the original HGVS. + + Returns a :class:`VepResolution`: ``consequences`` for the hits, and ``errored`` for HGVS whose VEP + or Recoder *request failed* (HTTP/transport error after retries) — those are unknown and must be + retried, never conflated with a genuine empty. An input in neither set was queried successfully and + yielded no consequence (a genuine empty). A failed batch does not abort the run: only that batch's + inputs are marked errored and processing continues. + """ + all_consequences: dict[str, str] = {} + errored: set[str] = set() + batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) + + # --- Phase 1: initial VEP pass --- + all_missing_hgvs: set[str] = set() + for batch_idx, batch in enumerate(batches): + try: + consequences = await get_functional_consequence(list(batch)) + # VEP rejected the batch (e.g. 400 for protein HGVS it cannot parse). Route all + # items to Variant Recoder — the input may still be resolvable via genomic recoding. + except requests.exceptions.HTTPError: + logger.warning( + msg=f"VEP returned an HTTP error for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); routing to Variant Recoder.", + extra=job_manager.logging_context(), + ) + all_missing_hgvs.update(batch) + continue + # Transport/network error — result unknown. Mark errored and do not route to Recoder. + except requests.exceptions.RequestException as exc: + logger.warning( + msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + + hit_consequences = {h: c for h, c in consequences.items() if c is not None} + all_consequences.update(hit_consequences) + all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) + + job_manager.update_progress( + int((batch_idx + 1) / len(batches) * 33), + 100, + f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + ) + + logger.info( + msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} HGVS strings require Variant Recoder fallback.", + extra=job_manager.logging_context(), + ) + + if not all_missing_hgvs: + return VepResolution(all_consequences, errored) + + # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- + recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) + semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) + completed_recoder_batches = 0 + + async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, list[str]]: + nonlocal completed_recoder_batches + async with semaphore: + result = await run_variant_recoder(batch) + completed_recoder_batches += 1 + job_manager.update_progress( + 33 + int(completed_recoder_batches / total * 33), + 100, + f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", + ) + return result + + total_recoder_batches = len(recoder_batch_list) + recoder_results = await asyncio.gather( + *[_recoder_with_semaphore(list(b), total_recoder_batches) for b in recoder_batch_list], + return_exceptions=True, + ) + + # A failed Recoder batch marks its inputs errored (they were misses; we still don't know them) and + # the run continues — one bad batch must not abort every allele's annotation. + hgvs_to_genomic: dict[str, list[str]] = {} + for batch, result in zip(recoder_batch_list, recoder_results): + if isinstance(result, BaseException): + logger.warning( + msg=f"Variant Recoder request failed for a batch of {len(batch)} HGVS; marking errored.", + exc_info=result, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + hgvs_to_genomic.update(result) + + logger.info( + msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", + extra=job_manager.logging_context(), + ) + + # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- + all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) + recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) + all_recoded_consequences: dict[str, str | None] = {} + errored_genomic: set[str] = set() + + for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): + try: + all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + except requests.exceptions.RequestException as exc: + logger.warning( + msg=f"VEP request failed for recoded batch {recoded_idx + 1}/{len(recoded_vep_batch_list)} " + f"({len(recoded_batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored_genomic.update(recoded_batch) + job_manager.update_progress( + 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), + 100, + f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", + ) + + # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. An + # original HGVS is a hit if any recoded form resolved; else errored if a recoded query failed and + # nothing resolved; else a genuine empty (Recoder ran, VEP found no consequence) — left out of both. + for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): + recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] + most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) + if most_severe: + all_consequences[original_hgvs] = most_severe + elif any(g in errored_genomic for g in recoded_hgvs_list): + errored.add(original_hgvs) + + return VepResolution(all_consequences, errored) + + @with_pipeline_management async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate VEP functional consequence predictions for all mapped variants in a ScoreSet. + """Link deduplicated alleles to their VEP functional consequence. - This function retrieves all mapped variants with a populated hgvs_assay_level field for a given - ScoreSet and submits them to the Ensembl VEP API in configurable batches. It handles fallback - to the Variant Recoder API for variants that cannot be processed by VEP directly. + Runs over the score set's current alleles (authoritative and RT-derived), submits each allele's + HGVS to VEP (with Variant Recoder fallback), and stores the most-severe consequence in a valid-time + :class:`VepAlleleConsequence`, superseding only on change. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the current-release skip and re-query every HGVS-bearing + allele. The linker still supersedes only on a value change, so a forced re-run of unchanged + data writes no new rows. Use for re-ingestion, to heal suspected corruption, or after editing + the VEP_CONSEQUENCES severity ordering (a change the release version cannot see). Args: ctx (dict): The job context dictionary. @@ -52,7 +242,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager (JobManager): Manager for job lifecycle and DB operations. Returns: - JobExecutionOutcome: Outcome with counts of processed, successful, and failed variants. + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ job = job_manager.get_job() @@ -62,6 +252,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Safely ignore mypy warnings here, as params were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] job_manager.save_to_context( { @@ -71,348 +262,156 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan "correlation_id": correlation_id, } ) - job_manager.update_progress(0, 100, "Starting VEP population.") - logger.info(msg="Started VEP population", extra=job_manager.logging_context()) - - mapped_variants = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.post_mapped.isnot(None), - ) - ).all() - - if not mapped_variants: - logger.warning( - msg=f"No mapped variants found for score set {score_set.urn}. Skipped VEP population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": 0, - "variants_with_consequences": 0, - "variants_without_consequences": 0, - "variants_recoder_failed": 0, - } - ) - - job_manager.save_to_context({"total_variants_to_process": len(mapped_variants)}) - logger.info( - msg=f"Found {len(mapped_variants)} mapped variants for VEP processing", - extra=job_manager.logging_context(), + job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") + logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) + + # One work-unit per allele (payload = HGVS; alleles without one are skipped). Events are allele-keyed, + # so each allele records its own event. + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=_vep_hgvs_payload, ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - mapped_variants_by_id = {mv.id: mv for mv in mapped_variants} - - # Extract HGVS strings; skip and annotate variants that have none. - hgvs_and_mapped_variant_id_pairs: list[tuple[str, int]] = [] - - for mapped_variant in mapped_variants: - if not mapped_variant.hgvs_assay_level: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={"error_message": "Mapped variant does not have an associated HGVS string."}, - ) - logger.debug("Mapped variant does not have an associated HGVS string.", extra=job_manager.logging_context()) - continue - - hgvs_and_mapped_variant_id_pairs.append((mapped_variant.hgvs_assay_level, mapped_variant.id)) # type: ignore - - batches = list(batched(hgvs_and_mapped_variant_id_pairs, _VEP_BATCH_SIZE)) - - job_manager.save_to_context({"vep_batches": len(batches)}) - logger.debug( - msg=f"Prepared {len(batches)} VEP batches ({_VEP_BATCH_SIZE} variants/batch)", - extra=job_manager.logging_context(), + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "absent_allele_count": 0, + "errored_allele_count": 0, + } ) - # --- Phase 1: Initial VEP pass --- - all_consequences: dict[str, str] = {} - all_missing_hgvs: set[str] = set() - - for batch_idx, batch in enumerate(batches): - logger.debug( - msg=f"Processing VEP batch {batch_idx + 1}/{len(batches)}", - extra=job_manager.logging_context(), - ) - - hgvs_strings, mapped_variant_ids = map(list, zip(*batch)) # type: ignore + num_alleles_with_hgvs = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) - consequences = await get_functional_consequence(hgvs_strings) - logger.debug( - msg=f"Received consequences for {len(consequences)} variants in VEP batch {batch_idx + 1}", + if not allele_data: + logger.warning( + msg="No current alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", extra=job_manager.logging_context(), ) - - # Only store variants where VEP returned an actual consequence string. A None value - # means VEP knew the variant but couldn't classify it — treat that the same as absent - # and route to Recoder so we have the best chance of getting a consequence. - hit_consequences = {h: c for h, c in consequences.items() if c is not None} - all_consequences.update(hit_consequences) - - missing_hgvs = set(hgvs_strings) - set(hit_consequences.keys()) - for hgvs, mapped_variant_id in zip(hgvs_strings, mapped_variant_ids): - if hgvs in missing_hgvs: - all_missing_hgvs.add(hgvs) - - progress_pct = int((batch_idx + 1) / len(batches) * 33) - job_manager.save_to_context( - { - "initial_vep_batches_processed": batch_idx + 1, - "missing_hgvs_count": len(all_missing_hgvs), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) + + all_allele_ids = set(allele_data.keys()) + + # The Ensembl release version-keys the run (coordinated software + transcript set + vocabulary). It + # is load-bearing for the skip below, so a failure here aborts the job rather than mis-versioning + # writes (the exception propagates to the job decorators). + ensembl_release = await get_ensembl_release() + job_manager.save_to_context({"ensembl_release": ensembl_release}) + + def alleles_at_current_release(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(VepAlleleConsequence.allele_id) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + .where(VepAlleleConsequence.source_version == ensembl_release) + ).all() ) - logger.info( - msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} variants require Variant Recoder fallback.", - extra=job_manager.logging_context(), + # Skip alleles already resolved at the current Ensembl release (they cannot change without a + # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES + # severity ordering we have since edited; the linker still supersedes only on a value change, so a + # forced no-op writes nothing. + already_current = set() if force else alleles_at_current_release(all_allele_ids) + hgvs_by_allele = {aid: allele_data[aid] for aid in allele_data if aid not in already_current} + unique_hgvs = sorted(set(hgvs_by_allele.values())) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_hgvs_to_query": len(unique_hgvs), + "force": force, + } ) - # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- - hgvs_to_genomic: dict[str, list[str]] = {} - recoder_missing_hgvs: set[str] = set() - - if all_missing_hgvs: - logger.info( - msg=f"Running Variant Recoder for {len(all_missing_hgvs)} HGVS strings", - extra=job_manager.logging_context(), - ) - - recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) - - logger.debug( - msg=f"Running {len(recoder_batch_list)} Variant Recoder batches with concurrency {_RECODER_CONCURRENCY}", - extra=job_manager.logging_context(), - ) - - semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) - completed_recoder_batches = 0 - - async def _recoder_with_semaphore(batch: list[str], batch_idx: int, total: int) -> dict[str, list[str]]: - nonlocal completed_recoder_batches - async with semaphore: - logger.debug( - msg=f"Starting Variant Recoder batch {batch_idx + 1}/{total} ({len(batch)} HGVS strings)", - extra=job_manager.logging_context(), - ) - result = await run_variant_recoder(batch) - completed_recoder_batches += 1 - logger.debug( - msg=f"Completed Variant Recoder batch {completed_recoder_batches}/{total} ({len(result)} variants recoded)", - extra=job_manager.logging_context(), - ) - progress_pct = 33 + int(completed_recoder_batches / total * 33) - job_manager.update_progress( - progress_pct, - 100, - f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", - ) - return result - - total_recoder_batches = len(recoder_batch_list) - recoder_results = await asyncio.gather( - *[ - _recoder_with_semaphore(list(recoder_batch), idx, total_recoder_batches) - for idx, recoder_batch in enumerate(recoder_batch_list) - ], - return_exceptions=True, - ) - - successful_batches = sum(1 for r in recoder_results if not isinstance(r, Exception)) - - first_exception = next((r for r in recoder_results if isinstance(r, Exception)), None) - if first_exception is not None: - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {str(first_exception)}", - extra=job_manager.logging_context(), - ) - raise first_exception - - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] - - job_manager.save_to_context( - { - "variant_recoder_batches_processed": len(recoder_batch_list), - "recoded_variants_count": len(hgvs_to_genomic), - } + verdicts: dict[int, VepLinkVerdict] = {} + errored_allele_ids: set[int] = set() + if unique_hgvs: + job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") + resolution = await _resolve_consequences(unique_hgvs, job_manager) + consequence_by_allele_id = {aid: resolution.consequences.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} + # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from empties. + errored_allele_ids = {aid for aid, hgvs in hgvs_by_allele.items() if hgvs in resolution.errored} + verdicts = link_vep_consequences_to_alleles( + job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() ) + job_manager.db.flush() + else: logger.info( - msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} variants successfully recoded.", + msg="All HGVS-bearing alleles are already resolved at the current Ensembl release; skipping VEP query.", extra=job_manager.logging_context(), ) + job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") - # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- - # hgvs_to_genomic maps original HGVS → list[str]; flatten to a deduplicated list of - # genomic strings before batching with VEP. - all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) - recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) - all_recoded_consequences: dict[str, str | None] = {} - - for recoded_vep_batch_idx, recoded_vep_batch in enumerate(recoded_vep_batch_list): - logger.debug( - msg=f"Processing recoded HGVS VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - extra=job_manager.logging_context(), + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, hgvs in allele_data.items(): + verdict = verdicts.get(allele_id) + if verdict is VepLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, ) - recoded_vep_consequences = await get_functional_consequence(recoded_vep_batch) - all_recoded_consequences.update(recoded_vep_consequences) - - progress_pct = 66 + int((recoded_vep_batch_idx + 1) / len(recoded_vep_batch_list) * 33) - job_manager.save_to_context( - { - "recoded_vep_batches_processed": recoded_vep_batch_idx + 1, - "recoded_consequences_count": len(all_recoded_consequences), - } + elif allele_id in already_current or verdict is VepLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed recoded VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - ) - - logger.info( - msg=f"Completed recoded VEP processing. {len(all_recoded_consequences)} recoded consequences retrieved.", - extra=job_manager.logging_context(), - ) - - # Map most-severe consequence from recoded genomic HGVS back to the original HGVS. - for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): - recoded_consequences_for_variant = [ - c for recoded_hgvs in recoded_hgvs_list if (c := all_recoded_consequences.get(recoded_hgvs)) - ] - - if recoded_consequences_for_variant: - most_severe = next( - (c for c in VEP_CONSEQUENCES if c in recoded_consequences_for_variant), - None, - ) - if most_severe: - all_consequences[original_hgvs] = most_severe - logger.debug( - msg=f"Selected most severe consequence '{most_severe}' for {original_hgvs}", - extra=job_manager.logging_context(), - ) - else: - logger.debug( - msg=f"Could not retrieve functional consequences for any recoded variants of {original_hgvs}", - extra=job_manager.logging_context(), - ) - - recoder_missing_hgvs = all_missing_hgvs - set(hgvs_to_genomic.keys()) - - # --- Phase 4: Annotate outcomes and update mapped variants in a single pass --- - - # HGVS strings that went through both VEP passes but still have no consequence. - all_processed_hgvs = {h for h, _ in hgvs_and_mapped_variant_id_pairs} - vep_failed_hgvs = all_processed_hgvs - set(all_consequences.keys()) - recoder_missing_hgvs - - variants_processed = 0 - variants_with_consequences = 0 - variants_without_consequences = 0 - variants_recoder_failed = 0 - - for hgvs_string, mapped_variant_id in hgvs_and_mapped_variant_id_pairs: - mapped_variant = mapped_variants_by_id.get(mapped_variant_id) # type: ignore - if mapped_variant is None: - continue - consequence = all_consequences.get(hgvs_string) - if consequence: - mapped_variant.vep_functional_consequence = consequence - mapped_variant.vep_access_date = date.today() - job_manager.db.add(mapped_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"functional_consequence": consequence}}, - ) - variants_with_consequences += 1 - logger.debug( - msg=f"Set consequence '{consequence}' for mapped variant {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in vep_failed_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "VEP could not determine a functional consequence for this variant, even after Variant Recoder fallback.", - }, - ) - variants_without_consequences += 1 - logger.debug( - msg=f"Recorded VEP failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in recoder_missing_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "Variant Recoder could not recode this HGVS string to a genomic equivalent.", - }, - ) - variants_recoder_failed += 1 - logger.debug( - msg=f"Recorded Variant Recoder failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), + elif allele_id in errored_allele_ids: + annotation_counts["errored_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=ensembl_release, + error_message="The VEP/Variant Recoder request for this allele failed; result unknown.", + metadata={"hgvs": hgvs}, ) + else: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "Variant was not classified by any VEP outcome branch. This is a bug.", - }, - ) - variants_without_consequences += 1 - logger.warning( - msg=f"Unexpected state: mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string}) was not classified by any outcome branch.", - extra=job_manager.logging_context(), + annotation_counts["absent_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=ensembl_release, + error_message="VEP found no functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": hgvs}, ) - variants_processed += 1 - annotation_manager.flush() - job_manager.db.flush() + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP functional consequence prediction for {variants_with_consequences}/{variants_processed} variants.", - ) - logger.info( - msg=f"Completed VEP prediction: {variants_with_consequences} with consequences, {variants_without_consequences} without, {variants_recoder_failed} recoder failed", - extra=job_manager.logging_context(), + ( + f"Completed VEP linkage: {annotation_counts['created_allele_count'] + annotation_counts['preexisting_allele_count']} linked, " + f"{annotation_counts['absent_allele_count']} absent (no result), " + f"{annotation_counts['errored_allele_count']} errored." + ), ) - + logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": variants_processed, - "variants_with_consequences": variants_with_consequences, - "variants_without_consequences": variants_without_consequences, - "variants_recoder_failed": variants_recoder_failed, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index ba9754a05..8b2b712f8 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -18,8 +18,6 @@ from mavedb.worker.jobs.external_services import ( link_gnomad_variants, poll_uniprot_mapping_jobs_for_score_set, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, refresh_clinvar_controls, submit_score_set_mappings_to_car, @@ -32,6 +30,7 @@ from mavedb.worker.jobs.variant_processing import ( create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, ) # All job functions for ARQ worker @@ -39,6 +38,7 @@ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -47,8 +47,6 @@ submit_uniprot_mapping_jobs_for_score_set, poll_uniprot_mapping_jobs_for_score_set, link_gnomad_variants, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, # Data management jobs refresh_materialized_views, @@ -102,6 +100,13 @@ "key": "map_variants_for_score_set", "type": JobType.VARIANT_MAPPING, }, + reverse_translate_variants_for_score_set: { + "dependencies": [], + "params": {"score_set_id": None, "correlation_id": None}, + "function": "reverse_translate_variants_for_score_set", + "key": "reverse_translate_variants_for_score_set", + "type": JobType.VARIANT_MAPPING, + }, submit_score_set_mappings_to_car: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, @@ -151,20 +156,6 @@ "key": "link_gnomad_variants", "type": JobType.MAPPED_VARIANT_ANNOTATION, }, - populate_hgvs_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_hgvs_for_score_set", - "key": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, - populate_variant_translations_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_variant_translations_for_score_set", - "key": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, populate_vep_for_score_set: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 4a2dc975e..40e636be6 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -46,11 +46,11 @@ # how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to # completion without being killed, while still catching a job that has genuinely hung. # 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose -# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (24h) so # the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and # actual work in sync. -PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often -RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling +PROGRESS_STALL_MINUTES = 30 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 1410 # Wall-clock backstop (23.5h), 30 min under the 24h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes diff --git a/src/mavedb/worker/jobs/variant_processing/__init__.py b/src/mavedb/worker/jobs/variant_processing/__init__.py index a6df09753..240de5e28 100644 --- a/src/mavedb/worker/jobs/variant_processing/__init__.py +++ b/src/mavedb/worker/jobs/variant_processing/__init__.py @@ -10,8 +10,10 @@ from .mapping import ( map_variants_for_score_set, ) +from .reverse_translation import reverse_translate_variants_for_score_set __all__ = [ "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", ] diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index efc8226b3..14d62fcdf 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -8,6 +8,7 @@ import asyncio import functools import logging +from collections import Counter from datetime import date from typing import Any @@ -23,13 +24,18 @@ ) from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS +from mavedb.lib.mapping.schema import MappingOutcome from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.allele import Allele as AlleleDbModel +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User @@ -130,7 +136,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Per-(target, alignment_level) QC records produced by the dcd-mapping API. # All records share the same tool_version because they come from a single run; - # we use that as the global ``mapping_api_version`` carried on each MappedVariant. + # we use that as the global ``mapping_api_version`` carried on each MappingRecord. target_mappings_payload = mapping_results.get("target_mappings") or [] tool_version = next( (tm.get("tool_version") for tm in target_mappings_payload if tm.get("tool_version")), @@ -178,7 +184,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan for annotation_layer in reference_metadata[target_gene_identifier]["layers"]: # ``annotation_layer`` arrives as a dcd-mapping wire code (``p``/``c``/``g``); # we persist metadata under the corresponding full-name enum value. - layer_name = AnnotationLayer.from_wire(annotation_layer).value + layer_name = SequenceLevel.from_wire(annotation_layer).value layer_premapped = reference_metadata[target_gene_identifier]["layers"][annotation_layer].get( "computed_reference_sequence" ) @@ -218,8 +224,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan continue target_gene_mapping = TargetGeneMapping( - target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.from_wire(level_value), + target_gene=target_gene, + alignment_level=SequenceLevel.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), reference_accession=tm.get("reference_accession"), @@ -255,12 +261,16 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan total_variants = len(mapped_scores) job_manager.save_to_context({"total_variants_to_process": total_variants}) - successful_mapped_variants = 0 + # Tally every record by its typed outcome; the mapped/failed/benign buckets are + # derived from this after the loop. Keeping all four keys preserves the + # intronic-vs-no-protein distinction in logs. + outcome_counts: Counter[MappingOutcome] = Counter() logger.info( f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id) + + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id, score_set_id=score_set.id) for mapped_score in mapped_scores: variant_urn = mapped_score.get("mavedb_id") variant = job_manager.db.scalars(select(Variant).where(Variant.urn == variant_urn)).one() @@ -268,23 +278,28 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"processing_variant": variant.id}) logger.debug(f"Processing variant {variant.id}.", extra=job_manager.logging_context()) - # there should only be one current mapped variant per variant id, so update old mapped variant to current = false + # Only allow one live MappingRecord per variant. The prior live record (if any) is + # superseded by the new version below via supersede_with, which retires it (cascading to + # its allele links) and inserts the new record under one timestamp. existing_mapped_variant = ( - job_manager.db.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + job_manager.db.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - existing_mapped_variant.current = False - job_manager.db.add(existing_mapped_variant) - logger.debug(msg="Set existing mapped variant to current = false.", extra=job_manager.logging_context()) - annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") - if annotation_was_successful: - successful_mapped_variants += 1 - job_manager.save_to_context({"successful_mapped_variants": successful_mapped_variants}) + # The typed outcome -- not allele presence -- decides success/benign/failure. + # Absent outcome means an older or malformed payload; fail fast. + raw_outcome = mapped_score.get("outcome") + if not raw_outcome: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} is missing its outcome." + ) + outcome = MappingOutcome(raw_outcome) + + outcome_counts[outcome] += 1 + job_manager.save_to_context({"outcome_counts": {o.value: n for o, n in outcome_counts.items()}}) # dcd-mapping guarantees both fields are set on every ScoreAnnotation, # including failed variants (the annotate step re-attributes failures to @@ -294,61 +309,121 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan score_alignment_level = mapped_score.get("alignment_level") if not score_target or not score_alignment_level: raise NonexistentMappingResultsError( - f"ScoreAnnotation for variant {variant_urn!r} is missing " - f"target_gene_identifier or alignment_level." + f"ScoreAnnotation for variant {variant_urn!r} is missing target_gene_identifier or alignment_level." ) + pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} + post_mapped_allele: dict = mapped_score.get("post_mapped") or {} + sequence_level = SequenceLevel.from_wire(score_alignment_level) + assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) + + # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping + # via (target_gene_identifier, alignment_level) -- including failed variants, which + # it re-attributes to the target's preferred layer. A miss implies a malformed payload. target_gene_mapping_row = target_gene_mapping_by_key.get((score_target, score_alignment_level)) - mapped_variant = MappedVariant( - pre_mapped=mapped_score.get("pre_mapped", null()), - post_mapped=mapped_score.get("post_mapped", null()), - hgvs_assay_level=get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + if target_gene_mapping_row is None: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} has no TargetGeneMapping for " + f"(target={score_target!r}, alignment_level={score_alignment_level!r})." + ) + + mapping_record = MappingRecord( variant_id=variant.id, - modification_date=date.today(), + vrs_digest=pre_mapped_allele.get("id"), + pre_mapped=pre_mapped_allele or None, + assay_level=sequence_level, + hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], - vrs_version=mapped_score.get("vrs_version", null()), + vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, - error_message=mapped_score.get("error_message", null()), - target_gene_mapping_id=target_gene_mapping_row.id if target_gene_mapping_row else None, - alignment_level=AnnotationLayer.from_wire(score_alignment_level), + target_gene_mapping_id=target_gene_mapping_row.id, + alignment_level=sequence_level, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), - current=True, ) - - annotation_manager.add_annotation( - variant_id=variant.id, # type: ignore - annotation_type=AnnotationType.VRS_MAPPING, - version=tool_version, - status=AnnotationStatus.SUCCESS if annotation_was_successful else AnnotationStatus.FAILED, - failure_category=None - if annotation_was_successful - else AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": mapped_score.get("error_message", null()), - "annotation_metadata": { - "mapped_assay_level_hgvs": get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), - }, + if existing_mapped_variant: + # Retire the prior record (cascading to its allele links) and insert this one under a + # single timestamp, so the prior valid_to equals the new valid_from with no gap. + existing_mapped_variant.supersede_with(job_manager.db, mapping_record) + logger.debug( + msg="Superseded prior mapping record and its allele links.", extra=job_manager.logging_context() + ) + else: + job_manager.db.add(mapping_record) + + # The mapper emits a MappingRecord for EVERY variant, so "a record exists" carries no + # signal — the signal is whether a real allele resulted. MAPPED yields an authoritative + # allele -> present. A benign absence (intronic, synonymous) produces a record but no + # allele: an informative biological negative -> absent. FAILED -> failed. `reason` reuses + # the MappingOutcome vocabulary so the intronic-vs-no-protein distinction survives. + if outcome is MappingOutcome.MAPPED: + disposition = Disposition.PRESENT + elif outcome.is_benign_absence: + disposition = Disposition.ABSENT + else: + disposition = Disposition.FAILED + + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=disposition, + reason=outcome.value, + source_version=tool_version, + metadata={ + "mapped_assay_level_hgvs": assay_level_hgvs, + "error_message": mapped_score.get("error_message"), }, - current=True, ) - job_manager.db.add(mapped_variant) + # Only variants with a post-mapped representation yield an authoritative Allele; + # failed and benign-absent variants get a MappingRecord but no linked allele. + if post_mapped_allele: + allele_draft = AlleleDbModel( + vrs_digest=post_mapped_allele["id"], + level=sequence_level, + hgvs_g=assay_level_hgvs if sequence_level == SequenceLevel.genomic else None, + hgvs_c=assay_level_hgvs if sequence_level == SequenceLevel.cdna else None, + hgvs_p=assay_level_hgvs if sequence_level == SequenceLevel.protein else None, + post_mapped=post_mapped_allele, + ) + authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) + job_manager.db.flush() + + # TODO#765: Mapping is not idempotent, so we must always create a new link. + job_manager.db.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=authoritative_allele.id, + is_authoritative=True, + ) + ) + logger.debug(msg="Linked mapped variant to authoritative allele.", extra=job_manager.logging_context()) + logger.debug(msg="Added new mapped variant to session.", extra=job_manager.logging_context()) + job_manager.db.flush() annotation_manager.flush() - if successful_mapped_variants == 0: + # Collapse the per-outcome tally into the three buckets the rest of the job reasons about. + mapped_count = outcome_counts[MappingOutcome.MAPPED] + failed_count = outcome_counts[MappingOutcome.FAILED] + skipped_count = sum(n for o, n in outcome_counts.items() if o.is_benign_absence) + + # State keys off genuine failures only: no failures -> complete (benign absences + # don't count); failures with nothing mapped -> failed; otherwise incomplete. + if failed_count == 0: + score_set.mapping_state = MappingState.complete + elif mapped_count == 0: score_set.mapping_state = MappingState.failed score_set.mapping_errors = {"error_message": "All variants failed to map."} - elif successful_mapped_variants < total_variants: - score_set.mapping_state = MappingState.incomplete else: - score_set.mapping_state = MappingState.complete + score_set.mapping_state = MappingState.incomplete job_manager.save_to_context( { - "successful_mapped_variants": successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "mapping_state": score_set.mapping_state.name, "mapping_errors": score_set.mapping_errors, "inserted_mapped_variants": len(mapped_scores), @@ -397,7 +472,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan logger.info(msg="Inserted mapped variants into db.", extra=job_manager.logging_context()) - if successful_mapped_variants == 0: + # Fail the job only on genuine failure with nothing mapped; all-benign is a success. + if mapped_count == 0 and failed_count > 0: logger.error(msg="No variants were successfully mapped.", extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -405,19 +481,27 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan data={ "score_set_id": score_set.id, "mapped_count": 0, - "unmapped_count": total_variants, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, }, failure_category=FailureCategory.VRS_MAPPING_FAILED, ) - logger.info(msg="Variant mapping job completed successfully.", extra=job_manager.logging_context()) + logger.info( + msg=( + f"Variant mapping job completed successfully: {mapped_count} mapped, " + f"{failed_count} failed, {skipped_count} skipped (intronic / no protein consequence)." + ), + extra=job_manager.logging_context(), + ) job_manager.db.flush() return JobExecutionOutcome.succeeded( data={ "score_set_id": score_set.id, - "mapped_count": successful_mapped_variants, - "unmapped_count": total_variants - successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, } ) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py new file mode 100644 index 000000000..a6cdfc257 --- /dev/null +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -0,0 +1,556 @@ +"""Reverse translation worker job — builds cross-level HGVS equivalence classes. + +For each mapped variant in a score set, calls construct_equivalent_variants from +the variant-annotation library to produce the coding/genomic HGVS candidates +encoding the same protein consequence, plus that protein consequence itself. Each +candidate is written as a non-authoritative Allele row linked to the existing +MappingRecord via MappingRecordAllele. + +The library returns the equivalence class as a list of ProjectionPair objects, +each one projection pair (a coding candidate and its deterministic genomic projection +— the same change at two levels). The job preserves that pairing by stamping both +links of a projection pair with a shared per-record ``projection_group`` id; the protein +apex is shared across all projection pairs and carries no group. Where a projection pair member +equals the record's authoritative (measured) allele, its group is folded onto the +existing authoritative link rather than duplicated, so the canonical c/g projection +is resolvable from the authoritative allele's group downstream. +""" + +import asyncio +import dataclasses +import functools +import logging +from collections import Counter +from datetime import date +from typing import Any, NamedTuple, Sequence + +from ga4gh.vrs.extras.translator import AlleleTranslator +from sqlalchemy import select +from variant_annotation import __version__ as variant_annotation_version +from variant_annotation.lib.accessions import looks_like_refseq_protein_accession +from variant_annotation.lib.translation import construct_equivalent_variants +from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode + +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.hgvs import extract_accession, strip_protein_prediction_parens +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele +from mavedb.lib.vrs_utils import translate_hgvs_to_variation +from mavedb.models.allele import Allele as AlleleDbModel +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.target_category import TargetCategory +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.jobs.utils.setup import validate_job_params +from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management +from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator, uta_transcript_source + +logger = logging.getLogger(__name__) + +# Job defaults: full coding equivalence class — every synonymous codon, indels included. +# wt_codon_mode "all" requires include_indels=True. +_DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { + "include_indels": True, + "wt_codon_mode": WtCodonMode.ALL, +} + + +class _TranscriptResolution(NamedTuple): + """Pairs a mapping record with its pre-UTA transcript info: gene_transcript if the mapper + supplied a cdna row, protein_accession if NP_→NM_ resolution is still needed. Both None + means the record will be skipped.""" + + rec: MappingRecord + variant: Variant + gene_transcript: str | None + protein_accession: str | None + target_gene_id: int | None + + +def _classify_skip( + resolution: _TranscriptResolution, category: TargetCategory | None +) -> tuple[EventReason, Disposition]: + """Classify why a record was skipped, returning its reason and event disposition. + + ``no_assay_level_hgvs`` — no input to translate, we could not ask → ``not_applicable``. + ``transcript_unresolved`` — protein-coding target with no transcript, a recoverable pipeline + gap → ``failed``. ``no_coding_transcript`` — non-coding target has no protein consequence, a + biological negative → ``absent``. + """ + if not resolution.rec.hgvs_assay_level: + return (EventReason.NO_ASSAY_LEVEL_HGVS, Disposition.NOT_APPLICABLE) + if category == TargetCategory.protein_coding: + return (EventReason.TRANSCRIPT_UNRESOLVED, Disposition.FAILED) + return (EventReason.NO_CODING_TRANSCRIPT, Disposition.ABSENT) + + +def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: + """Resolve RefSeq protein accessions (NP_/XP_) to their preferred coding transcripts via UTA. + + Protein-level mappings carry no cdna transcript in TargetGeneMapping, so reverse + translation falls back to the NP_→NM_ association in UTA. Connection is opened and + closed per-call to avoid leaking connections across long-lived worker jobs. + """ + if not protein_accessions: + return {} + + with uta_transcript_source() as client: + return { + pro_ac: transcript + for pro_ac in sorted(protein_accessions) + if (transcript := client.transcript_for_protein(pro_ac)) is not None + } + + +def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: + """Build a TranslationConfig from optional job-param overrides merged with job defaults. + + Overrides win over _DEFAULT_TRANSLATION_CONFIG; wt_codon_mode accepts string values and + is coerced to the enum. Raises ValueError for unknown fields, invalid modes, or invalid + combinations (e.g. wt_codon_mode != "none" without include_indels). + """ + config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} + + allowed_fields = {field.name for field in dataclasses.fields(TranslationConfig)} + unknown_fields = set(config_kwargs) - allowed_fields + if unknown_fields: + raise ValueError( + f"Unknown translation_config option(s): {', '.join(sorted(unknown_fields))}. " + f"Valid options: {', '.join(sorted(allowed_fields))}." + ) + + raw_mode = config_kwargs.get("wt_codon_mode") + if raw_mode is not None: + try: + config_kwargs["wt_codon_mode"] = WtCodonMode(raw_mode) + except ValueError: + valid_modes = ", ".join(repr(mode.value) for mode in WtCodonMode) + raise ValueError( + f"Invalid translation_config wt_codon_mode {raw_mode!r}. Valid values: {valid_modes}." + ) from None + + try: + return TranslationConfig(**config_kwargs) + except ValueError as exc: + raise ValueError(f"Invalid translation_config: {exc}") from exc + + +def _annotate_translation( + annotation_manager: AnnotationStatusManager, + variant_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CROSS_LEVEL_TRANSLATION event for an allele (the translation is a variant-level fact). + + The single choke point for RT's status writes. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CROSS_LEVEL_TRANSLATION, + variant_id=variant_id, + disposition=disposition, + reason=reason, + source_version=variant_annotation_version, + metadata=meta or None, + ) + + +@with_pipeline_management +async def reverse_translate_variants_for_score_set( + ctx: dict, job_id: int, job_manager: JobManager +) -> JobExecutionOutcome: + """Build the cross-level HGVS equivalence class for every mapped variant in the score set. + + For each current MappingRecord with an hgvs_assay_level, collapses to a ProteinConsequence + and expands to all coding/genomic HGVS candidates via the variant-annotation library. + Each candidate is written as a non-authoritative Allele linked to the MappingRecord. + + job_params: score_set_id (int), correlation_id (str), + translation_config (dict, optional) — TranslationConfig overrides. + """ + job = job_manager.get_job() + validate_job_params(["score_set_id", "correlation_id"], job) + + score_set_id: int = job.job_params["score_set_id"] # type: ignore[index] + correlation_id: str = job.job_params["correlation_id"] # type: ignore[index] + translation_config = _build_translation_config(job.job_params.get("translation_config")) # type: ignore[union-attr] + score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one() + + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "reverse_translate_variants_for_score_set", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) + job_manager.update_progress(0, 100, "Starting reverse translation job.") + logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) + + # Build {(target_gene_id, run date) -> NM_ transcript} from cdna TargetGeneMappings. + # TargetGeneMapping rows accumulate across re-maps (retired records still FK them), so we + # key by (target_gene_id, mapped_date) to bind each record to its own run's cdna row. + # All TargetGeneMappings in one job share a mapped_date; within a key, highest id wins. + # + # TODO#763: mapped_date is day-granular, so two re-maps on the same calendar day collide. + # A later same-day run that emits no cdna row can still bind the earlier run's stale + # transcript. Only a versioned run anchor closes this fully. + cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { + (target_gene_id, mapped_date): reference_accession + for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( + select( + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.reference_accession, + ) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where(TargetGene.score_set_id == score_set_id) + .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) + .where(TargetGeneMapping.reference_accession.isnot(None)) + .order_by(TargetGeneMapping.id) + ) + .tuples() + .all() + } + + # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. + # mapped_date from the joined (genomic) TargetGeneMapping anchors the cdna transcript lookup. + rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( + job_manager.db.execute( + select( + MappingRecord, + Variant, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + ) + .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + ) + .tuples() + .all() + ) + + annotation_counts: Counter[str] = Counter({"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + + if not rows: + logger.warning( + msg="No current and authoritative mapping records found for this score set.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) + + # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no + # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. + transcript_resolutions: list[_TranscriptResolution] = [] + protein_accessions: set[str] = set() + for rec, variant, target_gene_id, mapped_date in rows: + coding_accession = cdna_transcript_by_run.get((target_gene_id, mapped_date)) + protein_accession = None + if not coding_accession and rec.hgvs_assay_level is not None: + raw_accession = extract_accession(rec.hgvs_assay_level) + if looks_like_refseq_protein_accession(raw_accession): + protein_accession = raw_accession + protein_accessions.add(raw_accession) + + transcript_resolutions.append( + _TranscriptResolution(rec, variant, coding_accession, protein_accession, target_gene_id) + ) + + transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + + # Target category per gene — used to classify skips as recoverable (coding target, + # transcript unresolved) vs. correct (non-coding/regulatory, no protein consequence). + target_category_by_gene: dict[int, TargetCategory] = dict( + job_manager.db.execute( + select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ) + + # Build VariantInputs with the resolved coding transcript (p./c./g. all collapse to a + # ProteinConsequence on that transcript). Object identity is preserved so TranslationResult + # can be correlated back to its MappingRecord. Records with no transcript are skipped. + variant_inputs: list[Any] = [] + variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} + skipped_variants: list[_TranscriptResolution] = [] + for p in transcript_resolutions: + transcript = p.gene_transcript or ( + transcript_by_protein.get(p.protein_accession) if p.protein_accession else None + ) + if not transcript or not p.rec.hgvs_assay_level: + skipped_variants.append(p) + continue + + inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) + variant_inputs.append(inp) + variant_input_map[id(inp)] = (p.rec, p.variant) + + total = len(variant_inputs) + job_manager.save_to_context({"total_variants": total, "skipped_variants": len(skipped_variants)}) + job_manager.update_progress( + 10, + 100, + f"Prepared {total} variants for reverse translation ({len(skipped_variants)} skipped, no coding transcript).", + ) + logger.info( + msg=f"Running reverse translation for {total} variants ({len(skipped_variants)} skipped).", + extra=job_manager.logging_context(), + ) + + # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the + # default thread pool rather than the process pool to avoid pickling the port objects. + coordinates = WorkerCoordinateTranslator(ctx["hdp"]) + loop = asyncio.get_running_loop() + job_manager.update_progress(20, 100, "Running reverse translation subprocess.") + # WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, so pass a + # live UTA-backed source. codon_at is only touched in the post-subprocess WT-codon step, + # but the connection must outlive the executor call, so scope it around the whole call. + # (transcript_for_protein is redundant here -- the job already resolved every transcript + # and supplies it via VariantInput.transcript.) + with uta_transcript_source() as transcripts: + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) + + job_manager.update_progress(70, 100, "Writing translated alleles to database.") + logger.info( + msg=f"Translation complete: {len(results)} succeeded, {len(errors)} failed.", + extra=job_manager.logging_context(), + ) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set_id + ) + allele_translator = AlleleTranslator(ctx["seqrepo"]) + + current_record_ids = ( + select(MappingRecord.id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current.is_(True)) + ) + + # Defer linkage until all candidates are processed so the prior derived links can be + # superseded atomically — retire and insert share a timestamp with no gap. + new_links: list[MappingRecordAllele] = [] + + # The live authoritative links for the current records, keyed by (record_id, allele_id). + # Held as ORM objects (not a bare pair set) because the authoritative fold-in updates them in + # place: when an RT candidate's allele equals a record's authoritative (measured) allele, that + # link already exists (is_authoritative=True, projection_group NULL, written by the mapping job) + # and uq_mapping_record_alleles_live forbids a derived duplicate — so the candidate's group is + # stamped onto this existing link instead of inserting a new one. Modifying the loaded object + # emits an UPDATE on flush; supersede_live_where at the end only retires derived links, leaving + # the authoritative link (and its freshly stamped group) intact. + authoritative_links: dict[tuple[int, int], MappingRecordAllele] = { + (link.mapping_record_id, link.allele_id): link + for link in job_manager.db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ).all() + } + + for result in results: + rec, variant = variant_input_map[id(result.input)] + candidate_count = 0 + + # Equivalence generation may surface the same VRS object more than once. + # Dedup by vrs_digest per mapping record to avoid duplicate links. + seen_digests: set[str] = set() + failed_candidates: list[dict[str, str]] = [] + + # Flatten the projection pairs into a work list of (hgvs, level, field, projection_group) members, + # preserving the coding↔genomic pairing the library emits. Each ProjectionPair's coding and + # genomic members share the pair's per-record group id — its index in the equivalence class + # (0..N-1). hgvs_c is always present (the pair key); hgvs_g is None when the c→g projection + # failed, yielding a well-formed one-member (coding only) group rather than a desync. The + # group id is assigned once per pair here, so the two members are guaranteed to carry the + # same id even though they translate and link independently below. + members: list[tuple[str, SequenceLevel, str, int | None]] = [] + for group_id, pair in enumerate(result.projection_pairs): + members.append((pair.hgvs_c, SequenceLevel.cdna, "hgvs_c", group_id)) + if pair.hgvs_g is not None: + members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g", group_id)) + + # The protein consequence is the apex of the equivalence set — shared across every + # pair, a member of none — so it carries no group (None). Prediction parens + # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay + # inputs, where the protein is already the authoritative allele. + if result.hgvs_p: + members.append((strip_protein_prediction_parens(result.hgvs_p), SequenceLevel.protein, "hgvs_p", None)) + + for hgvs, level, hgvs_field, projection_group in members: + # A candidate may not be translatable (intronic projection, malformed expression). + # Track per-candidate failures; a variant fails only if *all* candidates fail. + try: + variation = translate_hgvs_to_variation(hgvs, allele_translator) + except Exception as e: + logger.warning( + msg=f"Failed to translate candidate HGVS to VRS: {hgvs} ({e})", + extra=job_manager.logging_context(), + ) + failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) + continue + + if variation.id in seen_digests: + continue + + seen_digests.add(variation.id) + draft_allele = AlleleDbModel( + vrs_digest=variation.id, + # exclude_none mirrors the mapper's serialization. + post_mapped=variation.model_dump(exclude_none=True), + level=level, + **{hgvs_field: hgvs}, # type: ignore[arg-type] + ) + allele = get_or_create_allele(job_manager.db, draft_allele) + job_manager.db.flush() + + authoritative_link = authoritative_links.get((rec.id, allele.id)) + if authoritative_link is not None: + # Authoritative fold-in: this member is the record's measured allele, already + # linked authoritatively. Stamp it's projection group to preserve the c↔g pairing, + # and skip creating a new derived link. + authoritative_link.projection_group = projection_group + continue + + new_links.append( + MappingRecordAllele( + mapping_record_id=rec.id, + allele_id=allele.id, + is_authoritative=False, + projection_group=projection_group, + ) + ) + candidate_count += 1 + + annotation_counts["alleles_created"] += candidate_count + annotation_metadata = { + "hgvs_input": result.input.hgvs, + # Serializable projection of the projection pairs (the pairing the links now encode). + "candidates": [ + {"hgvs_c": pair.hgvs_c, "hgvs_g": pair.hgvs_g, "variant_type": pair.variant_type} + for pair in result.projection_pairs + ], + "hgvs_p": result.hgvs_p, + "alleles_created": candidate_count, + "failed_candidates": failed_candidates, + } + + # No translatable candidates and failures mean the variant failed reverse translation. No + # failures and no candidates is a success with no alleles created. + if candidate_count == 0 and failed_candidates: + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_FAILED, + error_message="All candidate HGVS failed VRS translation.", + metadata=annotation_metadata, + ) + else: + annotation_counts["translated"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.TRANSLATED, + metadata=annotation_metadata, + ) + + # Supersede prior live derived links atomically. + # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints + # records; idempotent records would allow unchanged links to stay live. + MappingRecordAllele.supersede_live_where( + job_manager.db, + new_links, + MappingRecordAllele.is_authoritative.is_(False), + MappingRecordAllele.mapping_record_id.in_(current_record_ids), + ) + + # TODO#767: some non-substitution consequences (del/ins/delins/fs/ext) have no synonymous + # equivalence class, so they arrive here as TranslationErrors and are miscounted as + # FAILED. Classify by the protein consequence's edit type up front and map these to + # SKIPPED instead of pattern-matching engine error strings. + for error in errors: + _rec, variant = variant_input_map[id(error.input)] + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_ERROR, + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, + ) + + annotation_counts["skipped"] = len(skipped_variants) + for p in skipped_variants: + category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None + reason, disposition = _classify_skip(p, category) + _annotate_translation( + annotation_manager, + variant_id=p.variant.id, + disposition=disposition, + reason=reason, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, + ) + + annotation_manager.flush() + + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) + logger.info( + msg=( + f"Reverse translation complete: {annotation_counts['translated']} translated, " + f"{annotation_counts['failed']} failed, {annotation_counts['skipped']} skipped, " + f"{annotation_counts['alleles_created']} alleles created." + ), + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + + if annotation_counts["translated"] == 0 and annotation_counts["failed"] > 0: + logger.error( + msg="All variant reverse translations failed.", + extra=job_manager.logging_context(), + ) + return JobExecutionOutcome.failed( + reason="All variant reverse translations failed.", + data=outcome_data, + failure_category=FailureCategory.DATA_ERROR, + ) + + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py new file mode 100644 index 000000000..b715aace4 --- /dev/null +++ b/src/mavedb/worker/lib/translation_ports.py @@ -0,0 +1,68 @@ +"""Worker-side adapters for the variant_annotation translation ports. + +construct_equivalent_variants depends on two protocols — CoordinateTranslator +and TranscriptSource. This module supplies the worker's CoordinateTranslator +(WorkerCoordinateTranslator) and a factory (uta_transcript_source) for the +TranscriptSource, which is variant_annotation's own UTA-backed UtaClient. + +WorkerCoordinateTranslator defers AssemblyMapper initialization until the first +call — the hgvs library makes network calls on mapper construction that must not +fire in unit-test contexts where construct_equivalent_variants is mocked. +""" + +from __future__ import annotations + +import contextlib +import os +from collections.abc import Generator +from typing import Any + +from variant_annotation.lib.clients.uta import UtaClient, connect_uta + + +@contextlib.contextmanager +def uta_transcript_source() -> Generator[UtaClient]: + """Yield a UTA-backed TranscriptSource over a connection scoped to the block. + + Backs both the NP_→NM_ association lookup and the WT-codon read used by + WtCodonMode.ALL (TranscriptSource.codon_at). The connection is closed on exit, + so callers must use the client within the ``with`` block. + """ + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") + with contextlib.closing(connect_uta(uta_db_url)) as conn: + yield UtaClient(conn) + + +class WorkerCoordinateTranslator: + """CoordinateTranslator backed by the worker's HGVS data provider.""" + + def __init__(self, hdp: Any) -> None: + self._hdp = hdp + self._parser: Any = None + self._mapper: Any = None + + def _ensure_initialized(self) -> None: + if self._mapper is None: + import hgvs.assemblymapper + import hgvs.parser + + self._parser = hgvs.parser.Parser() + self._mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + ) + + def c_to_p(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_p(self._parser.parse(c_hgvs))) + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + self._ensure_initialized() + return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + + def c_to_g(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) diff --git a/src/mavedb/worker/settings/lifecycle.py b/src/mavedb/worker/settings/lifecycle.py index 54a0b4c76..5be66bd2d 100644 --- a/src/mavedb/worker/settings/lifecycle.py +++ b/src/mavedb/worker/settings/lifecycle.py @@ -9,7 +9,7 @@ from concurrent import futures -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy def standalone_ctx(): @@ -18,6 +18,7 @@ def standalone_ctx(): ctx["pool"] = futures.ProcessPoolExecutor() ctx["redis"] = None # Redis connection can be set up here if needed. ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} # Additional context setup can be added here as needed. @@ -37,6 +38,7 @@ async def shutdown(ctx): async def on_job_start(ctx): ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 3b0ab4e89..f205fae7e 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,13 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (24h) so ARQ is the *last* resort: # our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before # this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele # sets can legitimately run for hours, so a low ceiling would kill healthy work. # cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before # ARQ's hard timeout fires. -JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours +JOB_TIMEOUT_SECONDS = 24 * 60 * 60 # 24 hours class ArqWorkerSettings: diff --git a/tests/conftest.py b/tests/conftest.py index 34e366392..d2825fcdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,11 +95,11 @@ def session(postgresql): Base.metadata.create_all(bind=engine) # Create a unique index for the published_variants_materialized_view to - # enforce uniqueness on (variant_id, mapped_variant_id, score_set_id). This + # enforce uniqueness on (variant_id, mapping_record_id, score_set_id). This # allows us to test mat view refreshes that require this constraint. session.execute( text("""CREATE UNIQUE INDEX IF NOT EXISTS published_variants_mv_unique_idx - ON published_variants_materialized_view (variant_id, mapped_variant_id, score_set_id)"""), + ON published_variants_materialized_view (variant_id, mapping_record_id, score_set_id)"""), ) session.commit() diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py new file mode 100644 index 000000000..938d2062e --- /dev/null +++ b/tests/db/test_mixins.py @@ -0,0 +1,330 @@ +"""Unit tests for the ValidTime mixin. + +Tested against minimal, purpose-built models rather than any real MaveDB models. ``_VtParent`` is +a cascade-bearing entity (natural key ``key``); ``_VtChild`` is a leaf link (natural key +``(parent_id, tag)``). Both carry a partial unique index over their natural key +``WHERE valid_to IS NULL``, mirroring the contract real consumers must honor. + +Many checks pass an explicit ``at`` that differs from the DB clock: a single-transaction test cannot +otherwise distinguish a deliberate shared timestamp from the coincidental same-transaction handoff +the gap bug relied on, so the explicit ``at`` is what actually proves the timestamp is threaded. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import Column, ForeignKey, Index, Integer, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import aliased, declarative_base, relationship + +from mavedb.db.mixins import ValidTime + +VtBase = declarative_base() + +# Fixed, deterministic windows — far from the test's transaction clock so an accidental +# server_default (func.now()) stamp would be visibly wrong rather than coincidentally equal. +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +class _VtParent(ValidTime, VtBase): + __tablename__ = "vt_parents" + + id = Column(Integer, primary_key=True) + key = Column(Integer, nullable=False) + children = relationship("_VtChild", back_populates="parent") + + __retire_cascade__ = ("children",) + __table_args__ = (Index("uq_vt_parents_live", "key", unique=True, postgresql_where=text("valid_to IS NULL")),) + + +class _VtChild(ValidTime, VtBase): + __tablename__ = "vt_children" + + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey("vt_parents.id"), nullable=False) + tag = Column(Integer, nullable=False) + parent = relationship("_VtParent", back_populates="children") + + __table_args__ = ( + Index("uq_vt_children_live", "parent_id", "tag", unique=True, postgresql_where=text("valid_to IS NULL")), + ) + + +@pytest.fixture(autouse=True) +def vt_tables(session): + """Create the throwaway ValidTime tables on the test engine, drop them after. Autouse so every + test in the module gets the tables without naming the fixture in its signature.""" + engine = session.get_bind() + VtBase.metadata.create_all(bind=engine) + yield + # Release any locks the test's still-open transaction holds before dropping: a post-commit + # attribute access reopens a transaction holding ACCESS SHARE on these tables, and DROP TABLE + # would block on it forever (this fixture tears down while the session fixture is still open). + session.rollback() + VtBase.metadata.drop_all(bind=engine) + + +def _parent(session, key, *, valid_from=None): + p = _VtParent(key=key, valid_from=valid_from) if valid_from else _VtParent(key=key) + session.add(p) + session.commit() + return p + + +def _child(session, parent, tag, *, valid_from=None): + c = ( + _VtChild(parent_id=parent.id, tag=tag, valid_from=valid_from) + if valid_from + else _VtChild(parent_id=parent.id, tag=tag) + ) + session.add(c) + session.commit() + return c + + +class TestCurrentAndAsOf: + def test_fresh_row_is_current(self, session): + p = _parent(session, 1) + assert p.valid_to is None + assert p.current is True + + def test_retired_row_is_not_current(self, session): + p = _parent(session, 1) + p.retire(session, at=T1) + assert p.current is False + + def test_current_expression_filters_to_live_rows(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + session.commit() + + rows = session.scalars(select(_VtParent).where(_VtParent.current)).all() + assert [r.id for r in rows] == [live.id] + + def test_as_of_selects_rows_live_at_timestamp(self, session): + # Window [T0, T2): explicit valid_from and a retire at T2. + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def live_ids(ts): + return [r.id for r in session.scalars(select(_VtParent).where(_VtParent.as_of(ts))).all()] + + assert live_ids(T0 - timedelta(days=1)) == [] # before the window opens + assert live_ids(T0) == [p.id] # at valid_from (inclusive) + assert live_ids(T1) == [p.id] # inside the window + assert live_ids(T2) == [] # at valid_to (exclusive) + assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes + + +class TestLiveAt: + """``live_at(as_of)`` is the one-call ``as_of(ts) if ts else current`` — same result, but it also + works on an ``aliased()`` entity, which the raw ternary at call sites does too, but only clumsily.""" + + def test_live_at_none_is_current_on_instance(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + assert live.live_at(None) is True + assert retired.live_at(None) is False + + def test_live_at_ts_matches_the_window_on_instance(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + assert p.live_at(T0 - timedelta(days=1)) is False # before the window opens + assert p.live_at(T0) is True # at valid_from (inclusive) + assert p.live_at(T1) is True # inside the window + assert p.live_at(T2) is False # at valid_to (exclusive) + + def test_live_at_expression_agrees_with_current_and_as_of(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def ids(pred): + return [r.id for r in session.scalars(select(_VtParent).where(pred)).all()] + + assert ids(_VtParent.live_at(None)) == ids(_VtParent.current) == [] # retired -> not current + assert ids(_VtParent.live_at(T1)) == ids(_VtParent.as_of(T1)) == [p.id] + + def test_live_at_is_alias_aware(self, session): + # The point of live_at: it scopes to the alias, not the base table (a plain classmethod on an + # aliased entity is the trap this avoids). Both the compiled SQL and a query through the alias. + p = _parent(session, 1, valid_from=T0) + session.commit() + alias = aliased(_VtParent, name="pa") + assert "pa." in str(alias.live_at(T1)) + rows = session.scalars(select(alias).where(alias.live_at(T1))).all() + assert [r.id for r in rows] == [p.id] + + +class TestRetire: + def test_retire_closes_valid_to(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + assert c.valid_to == T1 + + def test_retire_is_idempotent(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + c.retire(at=T2) # already closed — keeps the original valid_to + assert c.valid_to == T1 + + def test_leaf_retire_needs_no_session(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire() # no cascade declared, so no session required + session.commit() + assert c.valid_to is not None + + def test_retire_cascades_to_live_child_links(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + p.retire(session, at=T1) + session.commit() + + assert p.valid_to == T1 + assert c1.valid_to == T1 # cascade closes the children at the same instant + assert c2.valid_to == T1 + + def test_cascade_retire_without_session_raises(self, session): + p = _parent(session, 1) + with pytest.raises(ValueError, match="cascade-retire"): + p.retire() # cascade declared but no session to issue the child UPDATE + + def test_cascade_runs_even_when_row_already_closed(self, session): + # A half-finished prior retire (parent closed, child not) must still converge. + p = _parent(session, 1) + c = _child(session, p, 1) + p.valid_to = T0 # simulate parent closed without its child + session.commit() + + p.retire(session, at=T1) # idempotent on the parent, but still cascades + session.commit() + assert p.valid_to == T0 # unchanged (idempotent) + assert c.valid_to == T1 # child still retired + + +class TestRetireLiveWhere: + def test_retires_matching_live_rows(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert c1.valid_to == T1 + assert c2.valid_to == T1 + + def test_leaves_already_retired_rows_untouched(self, session): + p = _parent(session, 1) + already = _child(session, p, 1) + already.retire(at=T0) + session.commit() + live = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert already.valid_to == T0 # retired row keeps its original valid_to + assert live.valid_to == T1 + + +class TestSupersedeWith: + def test_gap_free_handoff_with_explicit_at(self, session): + old = _parent(session, 1, valid_from=T0) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + # The retired valid_to equals the successor's valid_from exactly — no point-in-time hole. + assert old.valid_to == T1 + assert new.valid_from == T1 + + def test_default_at_handoff_is_gap_free(self, session): + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new) # at defaults to a single captured func.now() + session.commit() + + assert old.valid_to is not None + assert old.valid_to == new.valid_from + + def test_cascades_child_links_at_the_handoff_timestamp(self, session): + old = _parent(session, 1) + child = _child(session, old, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + assert old.valid_to == T1 + assert new.valid_from == T1 + assert child.valid_to == T1 # the old parent's child link retired under the same timestamp + + def test_respects_partial_unique_index_via_flush_ordering(self, session): + # old and new share natural key=1; supersede flushes the retire before inserting the + # successor, so the two are never simultaneously live and the unique index does not trip. + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() # would raise IntegrityError if both were live at the INSERT + + live = session.scalars(select(_VtParent).where(_VtParent.key == 1, _VtParent.current)).all() + assert [r.id for r in live] == [new.id] + + +class TestSupersedeLiveWhere: + def test_stamps_explicit_valid_from_on_inserts(self, session): + # Without flushing, valid_from would be None under the server_default path; supersede sets it + # explicitly, which is what keeps the handoff gap-free across a later-transaction flush. + new = _VtChild(parent_id=1, tag=1) + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == 1, at=T1) + assert new.valid_from == T1 + + def test_gap_free_handoff_with_reused_natural_key(self, session): + p = _parent(session, 1) + old = _child(session, p, 1) + new = _VtChild(parent_id=p.id, tag=1) # same natural key (parent_id, tag) + + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == p.id, at=T1) + session.commit() # reused key: retire executes before the insert, so the index holds + + assert old.valid_to == T1 + assert new.valid_from == T1 + live = session.scalars(select(_VtChild).where(_VtChild.parent_id == p.id, _VtChild.current)).all() + assert [r.id for r in live] == [new.id] + + def test_empty_new_rows_retires_the_scope(self, session): + # A re-run that produces nothing should still withdraw the prior live set. + p = _parent(session, 1) + old = _child(session, p, 1) + + _VtChild.supersede_live_where(session, [], _VtChild.parent_id == p.id, at=T1) + session.commit() + + assert old.valid_to == T1 + assert session.scalars(select(_VtChild).where(_VtChild.current)).all() == [] + + def test_refuses_a_cascade_bearing_class(self, session): + # Bulk supersede cannot fire __retire_cascade__, so it must refuse rather than orphan links. + with pytest.raises(ValueError, match="__retire_cascade__"): + _VtParent.supersede_live_where(session, [], _VtParent.key == 1) + + +class TestPartialUniqueIndexBackstop: + def test_two_live_rows_for_one_key_raise(self, session): + # The DB index is the loud backstop: forgetting to retire a predecessor before inserting its + # successor fails at flush instead of silently leaving two live rows. + session.add(_VtParent(key=1)) + session.add(_VtParent(key=1)) + with pytest.raises(IntegrityError): + session.commit() diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index faabe6d18..c7f62eaf3 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1825,14 +1825,14 @@ TEST_SAVED_CLINVAR_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "183058", "geneSymbol": "PTEN", "clinicalSignificance": "Likely benign", "clinicalReviewStatus": "criteria provided, multiple submitters, no conflicts", "dbName": "ClinVar", "dbVersion": "11_2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -1847,14 +1847,14 @@ TEST_SAVED_GENERIC_CLINICAL_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "ABC123", "geneSymbol": "BRCA1", "clinicalSignificance": "benign", "clinicalReviewStatus": "lots of convincing evidence", "dbName": "GenDB", "dbVersion": "2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -2191,7 +2191,7 @@ "faf95MaxAncestry": TEST_GNOMAD_FAF95_MAX_ANCESTRY, "creationDate": date.today().isoformat(), "modificationDate": date.today().isoformat(), - "recordType": "GnomADVariantWithMappedVariants", + "recordType": "GnomADVariantWithVariantLinks", "id": 1, # Presuming this is the only gnomAD variant in the database } diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py new file mode 100644 index 000000000..24b9c11f3 --- /dev/null +++ b/tests/helpers/util/annotation.py @@ -0,0 +1,130 @@ +"""Seeding helpers for the new-model annotation substrate. + +The serving layer (the lean whole-set view, the ``/variants/{urn}`` detail envelope, allele +annotations, and the score-set clinical-controls routes) reads a variant's mapped alleles and +their annotations through the ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` graph and +the per-source link tables (``ClinvarAlleleLink`` / ``GnomadAlleleLink`` / ``VepAlleleConsequence``) +— not the legacy ``MappedVariant`` association tables. These helpers build that graph in tests so a +score set's variants carry alleles and annotations the way the mapping/annotation pipeline would. + +``seed_mapping_record`` is the single reusable builder; higher-level helpers (e.g. +``link_clinical_controls_to_alleles``) compose it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional, Sequence, Union + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass +class AlleleSpec: + """One allele on a mapping record, plus the annotations that hang off it. + + Digests are the allele's identity in the serving layer, so they must be distinct within a test. + ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router + fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. + """ + + digest: str + level: str = "cdna" + is_authoritative: bool = False + clingen_allele_id: Optional[str] = None + hgvs_c: Optional[str] = None + hgvs_p: Optional[str] = None + hgvs_g: Optional[str] = None + post_mapped: Optional[dict] = None + vep_consequence: Optional[str] = None + projection_group: Optional[int] = None + clinvar_control_ids: Sequence[int] = field(default_factory=tuple) + gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) + + +def seed_mapping_record( + session: Session, + variant: Union[Variant, str], + *, + alleles: Sequence[AlleleSpec], + assay_level: str = "cdna", + hgvs_assay_level: Optional[str] = None, + mapping_api_version: str = "test.0.0", + pre_mapped: Optional[dict] = None, + valid_from: Optional[datetime] = None, +) -> MappingRecord: + """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. + + ``variant`` may be a ``Variant`` instance or its URN. ``pre_mapped`` sets the record's assayed-level + VRS (the flat ``preMapped`` field on the variant detail). When ``valid_from`` is set it stamps the + record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place + the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row + is live). Returns the created ``MappingRecord``. + """ + if isinstance(variant, str): + resolved = session.scalar(select(Variant).where(Variant.urn == variant)) + assert resolved is not None, f"variant with URN '{variant}' not found" + variant = resolved + + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version=mapping_api_version, + pre_mapped=pre_mapped, + ) + if valid_from is not None: + record.valid_from = valid_from + session.add(record) + session.commit() + + for spec in alleles: + allele = Allele( + vrs_digest=spec.digest, + level=spec.level, + clingen_allele_id=spec.clingen_allele_id, + hgvs_c=spec.hgvs_c, + hgvs_p=spec.hgvs_p, + hgvs_g=spec.hgvs_g, + post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, + ) + session.add(allele) + session.commit() + + links: list[Any] = [ + MappingRecordAllele( + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=spec.is_authoritative, + projection_group=spec.projection_group, + ) + ] + if spec.vep_consequence is not None: + links.append( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=spec.vep_consequence, + source_version="116", + access_date="2026-01-01", + ) + ) + links.extend(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=cid) for cid in spec.clinvar_control_ids) + links.extend(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gid) for gid in spec.gnomad_variant_ids) + + if valid_from is not None: + for link in links: + link.valid_from = valid_from + session.add_all(links) + + session.commit() + return record diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index b6d7801ae..35fd9e382 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -8,8 +8,6 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinicalControl as ClinicalControlDbModel -from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel @@ -24,6 +22,7 @@ TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, ) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.variant import mock_worker_variant_insertion @@ -204,60 +203,142 @@ def create_acc_score_set_with_variants( return score_set -def link_clinical_controls_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def seed_annotation_substrate(db, score_set, *, skip_first=False, pre_mapped=None): + """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. + + The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` + substrate (not the legacy ``MappedVariant``), so a variant needs a live record with an + authoritative ``post_mapped`` allele before a study result / statement can be constructed. This + mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is + deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) + + ``pre_mapped`` sets the assayed-level VRS on each record (the flat ``preMapped`` on the variant + detail). With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + "some variants were not mapped" cases). Returns the score set's variants in id order. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() - - # The first mapped variant gets the clinvar control, the second gets the generic control. - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) - mapped_variants[1].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 2)) - ) - - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() + for variant in variants[1:] if skip_first else variants: + seed_mapping_record( + db, + variant, + pre_mapped=pre_mapped, + alleles=[ + AlleleSpec( + digest=f"va-allele-{variant.id}", + is_authoritative=True, + clingen_allele_id=f"CA{variant.id}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + return variants -def link_clinvar_control_to_mapped_variant(db, score_set): - """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def seed_csv_substrate( + db, + score_set, + *, + assay_level="genomic", + hgvs_g=None, + hgvs_c=None, + hgvs_p=None, + vep_consequence=None, + clingen_allele_id=None, + gnomad_variant_ids=(), + clinvar_control_ids=(), + annotate="all", + valid_from=None, +): + """Seed every variant of a score set with a live mapping record on the allele substrate, shaped for the + CSV export reader (``get_score_set_variants_as_csv``). + + The authoritative allele sits at ``assay_level`` and carries the annotations the ``vep``/``clingen``/ + ``gnomad``/``clinvar`` namespaces read (VEP consequence, ClinGen id, gnomAD/ClinVar links). For a + nucleotide assay, a projection-group sibling at the other nucleotide level and a protein apex allele are + added so the ``post_mapped_hgvs_{g,c,p}`` triple reconstructs the same way the lean view does. ``hgvs_g``/ + ``hgvs_c``/``hgvs_p`` supply the canonical string at each level (the one at ``assay_level`` is also the + record's ``hgvs_assay_level``). ``annotate`` controls which variants get the authoritative-allele + annotations: ``"all"`` (every variant) or ``"first"`` (only the first, for "one linked control" cases). + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() + level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} + assay_hgvs = level_hgvs[assay_level] + + for index, variant in enumerate(variants): + annotated = annotate == "all" or index == 0 + specs = [ + AlleleSpec( + digest=f"csv-auth-{variant.id}", + level=assay_level, + is_authoritative=True, + projection_group=0, + hgvs_g=hgvs_g if assay_level == "genomic" else None, + hgvs_c=hgvs_c if assay_level == "cdna" else None, + hgvs_p=hgvs_p if assay_level == "protein" else None, + vep_consequence=vep_consequence if annotated else None, + clingen_allele_id=clingen_allele_id if annotated else None, + gnomad_variant_ids=tuple(gnomad_variant_ids) if annotated else (), + clinvar_control_ids=tuple(clinvar_control_ids) if annotated else (), + ) + ] + # Nucleotide assay: add the other-nucleotide projection sibling and the protein apex so the + # g/c/p triple reconstructs. A protein assay populates only the protein slot (no c/g fan-out). + if assay_level in ("genomic", "cdna"): + other_level = "cdna" if assay_level == "genomic" else "genomic" + if level_hgvs[other_level] is not None: + specs.append( + AlleleSpec( + digest=f"csv-sib-{variant.id}", + level=other_level, + projection_group=0, + hgvs_g=hgvs_g if other_level == "genomic" else None, + hgvs_c=hgvs_c if other_level == "cdna" else None, + ) + ) + if hgvs_p is not None: + specs.append(AlleleSpec(digest=f"csv-prot-{variant.id}", level="protein", hgvs_p=hgvs_p)) + + seed_mapping_record( + db, variant, assay_level=assay_level, hgvs_assay_level=assay_hgvs, alleles=specs, valid_from=valid_from + ) - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) + return variants - db.add(mapped_variants[0]) - db.commit() +def link_clinical_controls_to_alleles(db, score_set): + """Seed the new-model annotation graph for a score set's first two variants and link the two + seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. -def link_gnomad_variants_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) + The first variant's allele gets the ClinVar control (id 1), the second gets the generic control + (id 2) — mirroring the two-control shape the clinical-controls routes are asserted against. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() - # The first mapped variant gets the gnomAD variant. - mapped_variants[0].gnomad_variants.append( - db.scalar(select(GnomADVariantDbModel).where(GnomADVariantDbModel.id == 1)) + seed_mapping_record( + db, + variants[0], + alleles=[AlleleSpec(digest="clinical-control-allele-0", is_authoritative=True, clinvar_control_ids=[1])], + ) + seed_mapping_record( + db, + variants[1], + alleles=[AlleleSpec(digest="clinical-control-allele-1", is_authoritative=True, clinvar_control_ids=[2])], ) - - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() def mock_worker_vrs_mapping(client, db, score_set, alleles=True): diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index cb6949b4e..252471ca8 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -4,13 +4,13 @@ from sqlalchemy import select +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from mavedb.worker.jobs import ( create_variants_for_score_set, map_variants_for_score_set, ) -from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import ( TEST_CODING_LAYER, @@ -23,6 +23,16 @@ ) +# Placeholder reference accessions for qualifying target-relative HGVS in mock +# post-mapped output, keyed by the HGVS kind prefix (``c.``/``n.``/``g.``/``p.``). +_PLACEHOLDER_ACCESSIONS = { + "c.": "NM_999999.1", + "n.": "NR_999999.1", + "g.": "NC_999999.1", + "p.": "NP_999999.1", +} + + async def create_variants_in_score_set( session, mock_s3_client, score_df, count_df, mock_worker_ctx, variant_creation_run ): @@ -128,6 +138,10 @@ async def construct_mock_mapping_output( "target_gene_identifier": target.name, "alignment_level": layer, "preferred": idx == 0, + # The accession this level was aligned against — a transcript (NM_) + # only at the coding level. Reverse translation resolves its coding + # transcript hint from the cdna entry's reference_accession. + "reference_accession": _PLACEHOLDER_ACCESSIONS[f"{layer}."], "tool_name": "dcd-mapping", "tool_version": "pytest.0.0", "tool_parameters": {}, @@ -155,15 +169,26 @@ async def construct_mock_mapping_output( "alignment_level": default_alignment_level, } - # Don't alter HGVS strings in post mapped output. This makes it considerably - # easier to assert correctness in tests. + # Reuse the variant's own HGVS in post-mapped output to keep assertions simple. + # Real mapper post-mapped HGVS is always accession-qualified (it is mapped onto + # a reference sequence), so prefix a placeholder accession when the source HGVS + # is target-relative (a bare ``c.``/``n.``/``g.``/``p.`` with no accession). if with_post_mapped: - mapped_score["post_mapped"]["expressions"][0]["value"] = variant.hgvs_nt or variant.hgvs_pro + hgvs = variant.hgvs_nt or variant.hgvs_pro + if hgvs and ":" not in hgvs: + hgvs = f"{_PLACEHOLDER_ACCESSIONS.get(hgvs[:2], 'NM_999999.1')}:{hgvs}" + mapped_score["post_mapped"]["expressions"][0]["value"] = hgvs # Skip every other variant if not with_all_variants if not with_all_variants and idx % 2 == 0: mapped_score["post_mapped"] = {} + # Mirror the mapper's per-record outcome: both alleles present -> MAPPED, else + # FAILED. Tests needing benign outcomes set ``outcome`` explicitly afterward. + mapped_score["outcome"] = ( + "mapped" if mapped_score["pre_mapped"] and mapped_score["post_mapped"] else "failed" + ) + mapping_output["mapped_scores"].append(mapped_score) if not mapping_output["mapped_scores"]: diff --git a/tests/helpers/variant_shapes.py b/tests/helpers/variant_shapes.py index 1d60c7e26..555cd974c 100644 --- a/tests/helpers/variant_shapes.py +++ b/tests/helpers/variant_shapes.py @@ -38,6 +38,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Optional +from mavedb.lib.csv.specs import CsvMappedRow +from mavedb.lib.variants import get_id_from_post_mapped from tests.helpers.constants import ( TEST_VALID_POST_MAPPED_VRS_ALLELE, TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY, @@ -188,6 +190,26 @@ def _make_non_coding(mapped_variant) -> None: ] +def to_csv_mapped_row(mapped_variant: Any) -> CsvMappedRow: + """Adapt a mock ``MappedVariant`` (as built by :meth:`VariantShape.build`) into the ``CsvMappedRow`` + the CSV composer now consumes. + + Production resolves ``hgvs_g``/``hgvs_c``/``hgvs_p``/``vrs_digest`` at the fetch layer + (``mapped_hgvs_by_level`` and the substrate query), not in the composer itself. This mirrors that + resolution over a mock, so the shape list still exercises every ``post_mapped`` payload variant for + the one CSV column that still derives from it (``post_mapped_vrs_id``). + """ + return CsvMappedRow( + hgvs_g=mapped_variant.hgvs_g, + hgvs_c=mapped_variant.hgvs_c, + hgvs_p=mapped_variant.hgvs_p, + hgvs_assay_level=mapped_variant.hgvs_assay_level, + vrs_digest=get_id_from_post_mapped(mapped_variant.post_mapped) if mapped_variant.post_mapped else None, + vep_functional_consequence=mapped_variant.vep_functional_consequence, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + + def shape_ids() -> list[str]: """Shape names, for use as pytest parametrize ids.""" return [shape.name for shape in VARIANT_SHAPES] diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 29a056c67..34fde44b7 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,10 +5,13 @@ including mock objects with proper calibrations and configurations. """ +from types import SimpleNamespace from unittest.mock import Mock import pytest +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.vrs import vrs_object_from_mapped_variant from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, @@ -36,6 +39,32 @@ def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID return mapped_variant +def annotation_context_for(mapped_variant, subject_variant=None) -> VariantAnnotationContext: + """Build a DB-free ``VariantAnnotationContext`` around a mock mapped variant. + + The annotation builders read a context's ``variant`` (scores/calibrations), ``record`` (VRS-mapper + provenance), ``measured_allele`` (study-result focus + ClinGen IRI), and ``subject_variant`` (the VA + proposition subject). We synthesize the record/allele from the mock's mapping fields; the subject + defaults to the concrete measured variation, matching a non-projected variant. Mutating + ``context.variant`` mutates the underlying mock variant, since it is the same object. + """ + record = SimpleNamespace( + mapping_api_version=mapped_variant.mapping_api_version, + mapped_date=mapped_variant.mapped_date, + ) + measured_allele = SimpleNamespace( + post_mapped=mapped_variant.post_mapped, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + return VariantAnnotationContext( + variant=mapped_variant.variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant or vrs_object_from_mapped_variant(mapped_variant.post_mapped), + as_of=None, + ) + + @pytest.fixture def mock_mapped_variant(): """Override main fixture with properly configured mock for annotation tests.""" @@ -52,3 +81,25 @@ def mock_mapped_variant_with_functional_calibration_score_set(): def mock_mapped_variant_with_pathogenicity_calibration_score_set(): """Fixture for mock mapped variant with pathogenicity calibration score set.""" return create_mock_mapped_variant_with_pathogenicity_calibration_score_set(clingen_allele_id="CA123456") + + +@pytest.fixture +def mock_annotation_context(mock_mapped_variant) -> VariantAnnotationContext: + """A DB-free annotation context for an uncalibrated variant.""" + return annotation_context_for(mock_mapped_variant) + + +@pytest.fixture +def mock_annotation_context_with_functional_calibration_score_set( + mock_mapped_variant_with_functional_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a functional calibration.""" + return annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + + +@pytest.fixture +def mock_annotation_context_with_pathogenicity_calibration_score_set( + mock_mapped_variant_with_pathogenicity_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a pathogenicity calibration.""" + return annotation_context_for(mock_mapped_variant_with_pathogenicity_calibration_score_set) diff --git a/tests/lib/annotation/conftest_optional.py b/tests/lib/annotation/conftest_optional.py index 143317ad4..e0cca139a 100644 --- a/tests/lib/annotation/conftest_optional.py +++ b/tests/lib/annotation/conftest_optional.py @@ -1,6 +1,6 @@ from unittest.mock import Mock -from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME +from mavedb.lib.annotation.calibration import CALIBRATION_SCOPE_EXTENSION_NAME from mavedb.lib.permissions.principal import Principal from mavedb.models.enums.user_role import UserRole from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index cbfc19f2a..fba0957af 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -2,12 +2,15 @@ Tests for mavedb.lib.annotation.annotate module. This module tests the main annotation functions that create statements and study results -for variants, focusing on object structure and validation. +for variants, focusing on object structure and validation. The builders take a DB-free +``VariantAnnotationContext`` (assembled by the ``mock_annotation_context*`` fixtures); the real +substrate-backed context build is covered by the integration tests in ``test_util.py``. """ # ruff: noqa: E402 from copy import deepcopy +from unittest.mock import patch import pytest @@ -20,16 +23,23 @@ variant_pathogenicity_statement, variant_study_result, ) -from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal, scope_of +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from tests.lib.annotation.conftest import ( + admin_principal, + annotation_context_for, + make_private, + owner_principal, + scope_of, +) @pytest.mark.unit class TestVariantStudyResult: """Unit tests for variant study result creation.""" - def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): + def test_variant_study_result_creates_valid_result(self, mock_annotation_context): """Test that variant study result creates a valid result object.""" - result = variant_study_result(mock_mapped_variant) + result = variant_study_result(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -37,42 +47,42 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): def test_a_study_result_discloses_a_calibration_scope(self, mock_mapped_variant): # Emitted unconditionally so that a record with no scope is never ambiguous between "public" and # "produced before disclosure existed". - assert scope_of(variant_study_result(mock_mapped_variant)) == "public" + assert scope_of(variant_study_result(annotation_context_for(mock_mapped_variant))) == "public" @pytest.mark.unit class TestVariantFunctionalImpactStatement: """Unit tests for variant functional impact statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_functional_impact_statement(mock_mapped_variant) + result = variant_functional_impact_statement(mock_annotation_context) assert result is None def test_only_research_use_only_calibrations_returns_none( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_functional_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_functional_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_functional_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_functional_impact_statement(context) assert result is None - def test_valid_statement_creation(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_valid_statement_creation(self, mock_annotation_context_with_functional_calibration_score_set): """Test creating valid functional impact statement with proper structure.""" - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" @@ -84,38 +94,37 @@ def test_valid_statement_creation(self, mock_mapped_variant_with_functional_cali ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that research-use-only calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_indeterminate( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that variant not in any functional range gets INDETERMINATE classification.""" from unittest.mock import patch from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set + context = mock_annotation_context_with_functional_calibration_score_set # Mock functional_classification_of_variant to return None range (variant not in any range) with patch( "mavedb.lib.annotation.annotate.functional_classification_of_variant", return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ): - result = variant_functional_impact_statement(mapped_variant) + result = variant_functional_impact_statement(context) assert result is not None assert result.type == "Statement" @@ -126,32 +135,34 @@ def test_no_statement_is_built_from_a_private_calibration( self, mock_mapped_variant_with_functional_calibration_score_set ): """A private calibration's thresholds and baseline scores must not reach an anonymous caller.""" - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - assert variant_functional_impact_statement(mapped_variant) is None + assert variant_functional_impact_statement(context) is None def test_an_entitled_caller_receives_a_statement_from_a_private_calibration( self, mock_mapped_variant_with_functional_calibration_score_set ): """Viewer-scoped emission: an export shows each principal what that principal may see.""" - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - assert variant_functional_impact_statement(mapped_variant, principal=admin_principal()) is not None + assert variant_functional_impact_statement(context, principal=admin_principal()) is not None def test_a_public_statement_discloses_a_public_calibration_scope( self, mock_mapped_variant_with_functional_calibration_score_set ): # VA-Spec statements carry no stable id, so a viewer-scoped statement must say that it is one. - statement = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + statement = variant_functional_impact_statement( + annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + ) assert scope_of(statement) == "public" def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( self, mock_mapped_variant_with_functional_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - statement = variant_functional_impact_statement(mapped_variant, principal=admin_principal()) + statement = variant_functional_impact_statement(context, principal=admin_principal()) assert scope_of(statement) == "restricted" @@ -160,49 +171,52 @@ def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( class TestVariantPathogenicityStatement: """Unit tests for variant pathogenicity statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_pathogenicity_statement(mock_mapped_variant) + result = variant_pathogenicity_statement(mock_annotation_context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_pathogenicity_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = mock_annotation_context_with_pathogenicity_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_pathogenicity_statement(context) assert result is None def test_only_research_use_only_calibration_returns_none( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_no_acmg_classifications_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_acmg_classifications_returns_none( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test that statement returns None when no ACMG classifications exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Remove ACMG classifications from all calibrations - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: acmg_removed = [deepcopy(r) for r in calibration.functional_classifications] for functional_classification in acmg_removed: functional_classification["acmgClassification"] = None calibration.functional_classifications = acmg_removed - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_valid_pathogenicity_statement_creation( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test creating valid pathogenicity statement with proper structure.""" - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.proposition.type == "VariantPathogenicityProposition" @@ -226,27 +240,27 @@ def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_p ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that research-use-only pathogenicity calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_functional_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that functional annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid: no functional classifications @@ -254,20 +268,19 @@ def test_skips_invalid_calibrations_when_functional_annotation( # Valid: retain default functional classifications mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_pathogenicity_annotation( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that pathogenicity annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid calibration: no functional classifications @@ -276,17 +289,15 @@ def test_skips_invalid_calibrations_when_pathogenicity_annotation( # Valid calibration retained mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_uncertain_significance( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that variant not in any range gets UNCERTAIN_SIGNIFICANCE classification.""" from unittest.mock import patch @@ -295,7 +306,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Mock both classification functions to return None range (variant not in any range) with ( @@ -304,11 +315,11 @@ def test_variant_not_in_any_range_returns_uncertain_significance( return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ), patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), ), ): - result = variant_pathogenicity_statement(mapped_variant) + result = variant_pathogenicity_statement(context) assert result is not None assert result.type == "Statement" @@ -316,7 +327,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( assert result.classification.primaryCoding.code.root == "uncertain significance" def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Regression test: hasEvidenceItems on VariantPathogenicityEvidenceLine must be model instances. @@ -324,7 +335,7 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( VariantPathogenicityEvidenceLine validation to fail when reconstructing nested VRS objects (e.g. Allele with production genomic coordinates). Model instances must be stored directly. """ - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None for evidence_line in result.hasEvidenceLines: @@ -340,23 +351,23 @@ def test_no_statement_is_built_from_a_private_calibration( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): """A private calibration's ACMG criteria must not reach an anonymous caller.""" - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - assert variant_pathogenicity_statement(mapped_variant) is None + assert variant_pathogenicity_statement(context) is None def test_the_owner_receives_a_statement_from_their_private_calibration( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - assert variant_pathogenicity_statement(mapped_variant, principal=owner_principal()) is not None + assert variant_pathogenicity_statement(context, principal=owner_principal()) is not None def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - statement = variant_pathogenicity_statement(mapped_variant, principal=admin_principal()) + statement = variant_pathogenicity_statement(context, principal=admin_principal()) assert scope_of(statement) == "restricted" @@ -365,44 +376,49 @@ def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( class TestVariantHighestLevelAnnotation: """Unit tests for the highest-materialized-layer resolver used by the public data dump.""" - def test_study_result_when_uncalibrated(self, mock_mapped_variant): - result = variant_highest_level_annotation(mock_mapped_variant) + def test_study_result_when_uncalibrated(self, mock_annotation_context): + result = variant_highest_level_annotation(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" - def test_functional_statement_when_functional_only(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_functional_statement_when_functional_only( + self, mock_annotation_context_with_functional_calibration_score_set + ): # The functional calibration fixture has no ACMG classifications, so the variant qualifies for the # functional layer but not pathogenicity. - result = variant_highest_level_annotation(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "ExperimentalVariantFunctionalImpactProposition" def test_pathogenicity_statement_when_calibrated( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): - result = variant_highest_level_annotation(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "VariantPathogenicityProposition" - def test_none_when_unmapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - result = variant_highest_level_annotation(mock_mapped_variant) - assert result is None + def test_none_when_the_variant_has_nothing_to_annotate(self, mock_annotation_context): + """An unmapped variant yields no context at all, so it never reaches a builder. What this guards + is the other absence: one raised part-way through a build must degrade to None, not propagate.""" + with patch( + "mavedb.lib.annotation.annotate.can_annotate_variant_for_pathogenicity_evidence", + side_effect=MappingDataDoesntExistException("no mapping data"), + ): + assert variant_highest_level_annotation(mock_annotation_context) is None def test_degrades_to_a_study_result_when_the_calibration_is_private( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): # A study result reports the measured score, which publishing the score set did make public. The # variant is still described; only the calibration-derived interpretation is withheld. - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - result = variant_highest_level_annotation(mapped_variant) + result = variant_highest_level_annotation(context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -410,9 +426,9 @@ def test_degrades_to_a_study_result_when_the_calibration_is_private( def test_reaches_the_statement_layer_for_an_entitled_caller( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - result = variant_highest_level_annotation(mapped_variant, principal=admin_principal()) + result = variant_highest_level_annotation(context, principal=admin_principal()) assert result is not None assert result.type != "ExperimentalVariantFunctionalImpactStudyResult" diff --git a/tests/lib/annotation/test_calibration.py b/tests/lib/annotation/test_calibration.py new file mode 100644 index 000000000..576f8d39f --- /dev/null +++ b/tests/lib/annotation/test_calibration.py @@ -0,0 +1,653 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.calibration — calibration eligibility + strongest-evidence selection.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.calibration import ( + calibrations_available_for_annotation, + score_calibration_may_be_used_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) +from mavedb.lib.permissions.principal import Principal +from tests.lib.annotation.conftest import admin_principal, make_private + + +def _has_calibrations_for_annotation(*args, **kwargs) -> bool: + """Whether any calibration survived both the eligibility and visibility checks. + + ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below assert + only on whether the list was empty. + """ + return bool(calibrations_available_for_annotation(*args, **kwargs)) + + +@pytest.mark.unit +class TestScoreCalibrationMayBeUsedForAnnotation: + def test_returns_false_for_research_use_only_when_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=False, + ) + is False + ) + + def test_returns_true_for_research_use_only_when_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=True, + ) + is True + ) + + def test_returns_false_when_functional_classifications_missing( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.functional_classifications = [] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False + + def test_returns_false_for_pathogenicity_without_acmg_classifications( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for functional_classification in acmg_removed: + functional_classification["acmgClassification"] = None + calibration.functional_classifications = acmg_removed + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False + + def test_returns_true_for_pathogenicity_with_any_acmg_classification( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True + + +@pytest.mark.unit +class TestCalibrationsAvailableForAnnotation: + """ + Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. + This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. + """ + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_annotation_context, kind): + mock_annotation_context.variant.score_set.score_calibrations = None + assert _has_calibrations_for_annotation(mock_annotation_context, kind) is False + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_annotation_context, kind): + mock_annotation_context.variant.score_set.score_calibrations = [] + assert _has_calibrations_for_annotation(mock_annotation_context, kind) is False + + @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( + self, mock_annotation_context_with_functional_calibration_score_set, annotation_type + ): + """Test that research use only calibrations are excluded by default.""" + for ( + calibration + ) in mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations: + calibration.research_use_only = True + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, annotation_type + ) + is False + ) + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( + self, kind, context_fixture, request + ): + """Test that research use only calibrations are included when explicitly allowed.""" + context = request.getfixturevalue(context_fixture) + for calibration in context.variant.score_set.score_calibrations: + calibration.primary = False + calibration.research_use_only = True + + assert _has_calibrations_for_annotation(context, kind, allow_research_use_only_calibrations=True) is True + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( + self, kind, context_fixture, request + ): + context = request.getfixturevalue(context_fixture) + + for calibration in context.variant.score_set.score_calibrations: + calibration.functional_classifications = None + + assert _has_calibrations_for_annotation(context, kind) is False + + def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( + self, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + for fr in acmg_classification_removed: + fr["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is False + ) + + def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( + self, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + acmg_classification_removed[0]["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is True + ) + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( + self, kind, context_fixture, request + ): + context = request.getfixturevalue(context_fixture) + + assert _has_calibrations_for_annotation(context, kind) is True + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( + self, mock_annotation_context_with_functional_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for functional annotation.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" + calibrations = ( + mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is True + ) + + def test_score_range_check_handles_mixed_functional_classifications( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test behavior when some calibrations have functional classifications and some don't.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[1].functional_classifications = None + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is True + ) + + def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + for calibration in calibrations: + if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: + acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for fc in acmg_classification_removed: + if "acmgClassification" in fc: + fc["acmgClassification"] = None + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "pathogenicity" + ) + is False + ) + + def test_functional_annotation_with_empty_functional_classifications_list( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test that functional annotation fails when functional classifications list is empty.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + for calibration in calibrations: + calibration.functional_classifications = [] + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is False + ) + + +@pytest.mark.unit +class TestCalibrationAvailabilityIsScopedToTheCaller: + """A calibration's READ rule is stricter than its score set's, so publishing a score set does not + publish its calibrations. These cases exist because this function once read + ``score_set.score_calibrations`` directly and handed private calibrations to anyone. + """ + + def test_a_private_calibration_is_not_available_for_annotation( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + == [] + ) + + def test_omitting_the_principal_withholds_rather_than_widens( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + # The whole design rests on an omitted principal meaning "the public". Passing an explicitly + # anonymous principal and passing none at all must agree. + make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + + assert calibrations_available_for_annotation(context, "functional") == calibrations_available_for_annotation( + context, "functional", principal=Principal() + ) + + def test_an_entitled_caller_still_receives_a_private_calibration( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional", principal=admin_principal() + ) + != [] + ) + + def test_a_public_calibration_is_still_available_to_anyone( + self, mock_annotation_context_with_functional_calibration_score_set + ): + # The counterweight: withholding private calibrations must not withhold what publishing released. + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + != [] + ) + + +@pytest.mark.unit +class TestSelectStrongestFunctionalCalibrationUnit: + """Unit tests for select_strongest_functional_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_annotation_context): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_functional_calibration(mock_annotation_context, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that single calibration is returned.""" + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_returns_first_when_all_agree(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that first calibration is returned when all have same classification.""" + context = mock_annotation_context_with_functional_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] # Should return the first one + + def test_defaults_to_normal_on_conflict(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that normal classification is preferred when there are conflicts.""" + from unittest.mock import MagicMock, patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + context = mock_annotation_context_with_functional_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different classifications + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + side_effect=[ + (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), + (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), + ], + ): + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + # Should return the normal classification (second one) + assert calibration == calibration2 + assert functional_range.label == "Normal" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_annotation_context_with_functional_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + # Mock to return None range but INDETERMINATE classification (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), + ): + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None + + +@pytest.mark.unit +class TestSelectStrongestPathogenicityCalibrationUnit: + """Unit tests for select_strongest_pathogenicity_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_annotation_context): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_pathogenicity_calibration(mock_annotation_context, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that single calibration is returned.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_selects_strongest_evidence_strength( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that calibration with strongest evidence is selected.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different evidence strengths + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ( + MagicMock(label="Strong"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.VERY_STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return the one with VERY_STRONG evidence + assert calibration == calibration2 + assert functional_range.label == "Strong" + + def test_defaults_to_uncertain_on_tie(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength but different criteria (pathogenic vs benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Pathogenic"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration but None for range to indicate uncertain significance + assert calibration == calibration1 + assert functional_range is None + + def test_returns_classification_when_all_tied_are_same_type( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that classification is returned normally when all tied candidates are the same type.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength and same type of criteria (both benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Benign1"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign2"), + VariantPathogenicityEvidenceLine.Criterion.BP1, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration with its range (no conflict, so normal classification) + assert calibration == calibration1 + assert functional_range.label == "Benign1" + + def test_handles_none_evidence_strength(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that None evidence strength is handled correctly.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock with None and actual strength + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return the one with actual strength (second) + assert calibration == calibration2 + assert functional_range.label == "Moderate" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + # Mock to return None range for all calibrations (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None diff --git a/tests/lib/annotation/test_classification.py b/tests/lib/annotation/test_classification.py index 82cd90684..fdbaf23cb 100644 --- a/tests/lib/annotation/test_classification.py +++ b/tests/lib/annotation/test_classification.py @@ -63,7 +63,7 @@ def test_normal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.normal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -76,7 +76,7 @@ def test_abnormal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.ABNORMAL @@ -89,7 +89,7 @@ def test_indeterminate_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.not_specified) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -104,7 +104,7 @@ def test_variant_not_in_any_range(self): ) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -122,7 +122,7 @@ def test_multiple_ranges_returns_first_match(self): second_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -134,7 +134,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -143,7 +143,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -152,7 +152,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) @pytest.mark.unit @@ -168,7 +168,7 @@ def test_valid_pathogenicity_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -186,7 +186,7 @@ def test_moderate_plus_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) # Verify the mapped evidence strength is MODERATE, not MODERATE_PLUS assert result[0] == pathogenicity_range @@ -201,7 +201,7 @@ def test_none_acmg_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification=None) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -216,7 +216,7 @@ def test_variant_not_in_any_range(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification, variant_in_range=False) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -229,7 +229,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -238,7 +238,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -247,7 +247,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_moderate_plus_evidence_strength_maps_to_moderate(self): """Test that MODERATE_PLUS evidence strength maps to Moderate.""" @@ -258,7 +258,7 @@ def test_moderate_plus_evidence_strength_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -282,7 +282,7 @@ def test_various_valid_acmg_combinations(self, criterion, strength): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion[criterion] @@ -301,7 +301,7 @@ def test_multiple_ranges_returns_first_match(self): score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 diff --git a/tests/lib/annotation/test_conformance.py b/tests/lib/annotation/test_conformance.py index ed83878ff..965951313 100644 --- a/tests/lib/annotation/test_conformance.py +++ b/tests/lib/annotation/test_conformance.py @@ -26,7 +26,7 @@ create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids -from tests.lib.annotation.conftest import scope_of +from tests.lib.annotation.conftest import annotation_context_for, scope_of # Each surface is paired with the factory that can actually exercise it: a statement built on a score # set with no calibrations returns None for every shape, which would test nothing. @@ -60,7 +60,7 @@ def test_annotation_round_trips_or_is_none(self, surface, shape): """Never raises, and anything emitted can be read back.""" _, annotate, factory = surface - annotation = annotate(shape.build(factory)) + annotation = annotate(annotation_context_for(shape.build(factory))) if annotation is None: pytest.skip(f"{shape.name} produces no annotation on this surface") @@ -71,7 +71,7 @@ def test_calibration_scope_survives_the_round_trip(self, surface, shape): between "public" and "produced before disclosure existed". It has to come back too.""" _, annotate, factory = surface - annotation = annotate(shape.build(factory)) + annotation = annotate(annotation_context_for(shape.build(factory))) if annotation is None: pytest.skip(f"{shape.name} produces no annotation on this surface") @@ -82,5 +82,7 @@ def test_calibration_scope_survives_the_round_trip(self, surface, shape): def test_every_surface_emits_something_for_at_least_one_shape(): """Guards the suite above: a surface that returned None everywhere would silently skip every case.""" for name, annotate, factory in ANNOTATION_SURFACES: - emitted = [shape.name for shape in VARIANT_SHAPES if annotate(shape.build(factory)) is not None] + emitted = [ + shape.name for shape in VARIANT_SHAPES if annotate(annotation_context_for(shape.build(factory))) is not None + ] assert emitted, f"{name} emitted nothing for any shape; its conformance cases are all skips" diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py new file mode 100644 index 000000000..e75256512 --- /dev/null +++ b/tests/lib/annotation/test_context.py @@ -0,0 +1,108 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.context — the VA proposition-subject grain (Slice 5.1).""" + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation + +from mavedb.lib.annotation.context import variant_annotation_context +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +@pytest.mark.integration +class TestVariantAnnotationContextSubject: + """variant_annotation_context — the VA *proposition* subject grain (Slice 5.1). + + The subject always anchors on the **measured** allele. A protein assay unfurls the full equivalence + class of encoders; a nucleotide assay keeps its precise coordinate partner + protein consequence and + excludes the sibling encoders (distinct variants) that the reverse-translation fan left on the record. + Either way the subject is a Cat-VRS ``CategoricalVariant`` when a projection member exists; it falls + back to the concrete measured ``MolecularVariation`` when the categorical can't be assembled. + """ + + def test_protein_assay_subject_is_a_categorical_variant(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="protein", + alleles=[ + AlleleSpec( + digest="prot", level="protein", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="cdna", level="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + + def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_change( + self, session, setup_lib_db_with_mapped_variant + ): + """Anchored on the measured nt change: keep its projection_group coordinate partner + the protein + consequence, exclude the sibling encoder (a distinct variant, different projection group).""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + projection_group=0, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + # The measured change's precise coordinate partner (same projection group). + AlleleSpec( + digest="gen", level="genomic", projection_group=0, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + # A sibling encoder of the same protein consequence — distinct variant, other group: excluded. + AlleleSpec( + digest="sibling", level="cdna", projection_group=1, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="prot", level="protein", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + # defining cdna + gen partner + protein apex; the sibling encoder is dropped. + assert len(context.subject_variant.members) == 3 + + def test_single_allele_subject_falls_back_to_molecular_variation(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele (no projection member) → the categorical build yields the concrete allele.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, MolecularVariation) + assert not isinstance(context.subject_variant, CategoricalVariant) + + def test_unmapped_variant_has_no_context(self, session, setup_lib_db_with_mapped_variant): + """No live mapping record on the new substrate → no annotation context.""" + context = variant_annotation_context(session, setup_lib_db_with_mapped_variant.variant) + + assert context is None diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 51597db37..60990baf6 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -23,14 +23,17 @@ mavedb_score_calibration_contribution, mavedb_vrs_contribution, ) +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_resource_with_dates, create_mock_score_calibration, create_mock_user, ) +from tests.lib.annotation.conftest import annotation_context_for @pytest.mark.unit @@ -85,14 +88,14 @@ class TestMavedbVrsContributionUnit: def test_returns_contribution_object(self): """Test that function returns proper Contribution object.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) def test_has_correct_name_and_description(self): """Test that contribution has correct name and description.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.name == "MaveDB VRS Mapper" assert contribution.description == "Contribution from the MaveDB VRS mapping software" @@ -100,7 +103,7 @@ def test_has_correct_name_and_description(self): def test_has_correct_activity_type(self): """Test that contribution has correct activity type.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.activityType == "human genome sequence mapping process" @@ -108,14 +111,14 @@ def test_uses_mapped_variant_date(self): """Test that contribution uses mapped variant date.""" test_date = datetime(2024, 3, 15, 12, 0, 0) mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.date == test_date def test_has_contributor(self): """Test that contribution has a contributor.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.contributor is not None @@ -123,7 +126,7 @@ def test_has_contributor(self): def test_various_api_versions(self, api_version): """Test function works with various API versions.""" mapped_variant = create_mock_mapped_variant(mapping_api_version=api_version) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) assert contribution.name == "MaveDB VRS Mapper" @@ -316,7 +319,12 @@ class TestContributionIntegration: def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_mapped_variant): """Test contribution creation from persisted SQLAlchemy objects.""" mapped_variant = setup_lib_db_with_mapped_variant - vrs_contribution = mavedb_vrs_contribution(mapped_variant) + # The VRS contribution derives from the record's mapper provenance, not the subject; hand the + # context an explicit subject so it doesn't depend on this fixture's minimal post_mapped. + context = annotation_context_for( + mapped_variant, subject_variant=vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + ) + vrs_contribution = mavedb_vrs_contribution(context) creator = session.query(User).first() score_set = mapped_variant.variant.score_set @@ -350,7 +358,7 @@ def test_all_contributions_return_consistent_structure(self): api_contrib = mavedb_api_contribution() mapped_variant = create_mock_mapped_variant() - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) calibration = create_mock_score_calibration() cal_contrib = mavedb_score_calibration_contribution(calibration) @@ -376,7 +384,7 @@ def test_date_formatting_consistency(self): # VRS contribution mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert vrs_contrib.date == test_date # Score calibration contribution diff --git a/tests/lib/annotation/test_document.py b/tests/lib/annotation/test_document.py index 0a9b0ac86..d87e9b227 100644 --- a/tests/lib/annotation/test_document.py +++ b/tests/lib/annotation/test_document.py @@ -8,6 +8,7 @@ """ import urllib.parse +from types import SimpleNamespace import pytest @@ -19,8 +20,7 @@ from mavedb.lib.annotation.document import ( experiment_as_iri, experiment_to_document, - mapped_variant_as_iri, - mapped_variant_to_document, + measured_allele_as_iri, score_calibration_as_document, score_set_as_iri, score_set_to_document, @@ -81,42 +81,24 @@ def test_score_set_to_document(self, mock_score_set): @pytest.mark.unit -class TestMappedVariantDocumentFunctions: - """Unit tests for mapped variant document creation functions.""" +class TestMeasuredAlleleDocumentFunctions: + """Unit tests for measured-allele IRI generation.""" - def test_mapped_variant_as_iri(self, mock_mapped_variant): - """Test IRI generation for mapped variants with ClinGen allele ID.""" - expected_iri_root = ( - f"https://mavedb.org/variant/{urllib.parse.quote_plus(mock_mapped_variant.clingen_allele_id)}" - ) - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri(self): + """Test IRI generation for a measured allele with a ClinGen allele ID.""" + allele = SimpleNamespace(clingen_allele_id="CA123456") + expected_iri_root = f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}" + result = measured_allele_as_iri(allele) assert result.root == expected_iri_root - def test_mapped_variant_as_iri_no_caid(self, mock_mapped_variant): - """Test IRI generation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri_no_caid(self): + """Test IRI generation for a measured allele without a ClinGen allele ID returns None.""" + allele = SimpleNamespace(clingen_allele_id=None) + result = measured_allele_as_iri(allele) assert result is None - def test_mapped_variant_to_document(self, mock_mapped_variant): - """Test document creation for mapped variants with ClinGen allele ID.""" - document = mapped_variant_to_document(mock_mapped_variant) - - assert document.id == mock_mapped_variant.variant.urn - assert document.name == "MaveDB Mapped Variant" - assert document.documentType == "mapped genomic variant description" - assert len(document.urls) > 0 - assert mapped_variant_as_iri(mock_mapped_variant).root in document.urls - - def test_mapped_variant_to_document_no_caid(self, mock_mapped_variant): - """Test document creation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - document = mapped_variant_to_document(mock_mapped_variant) - - assert document is None - @pytest.mark.unit class TestVariantDocumentFunctions: diff --git a/tests/lib/annotation/test_eligibility.py b/tests/lib/annotation/test_eligibility.py new file mode 100644 index 000000000..9c8f2ff47 --- /dev/null +++ b/tests/lib/annotation/test_eligibility.py @@ -0,0 +1,157 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.eligibility — variant annotatability predicates.""" + +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.eligibility import ( + _can_annotate_variant_base_assumptions, + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.lib.annotation.conftest import admin_principal, annotation_context_for, make_private + + +@pytest.mark.unit +class TestBaseAnnotationAssumptionsUnit: + def test_base_assumption_check_returns_false_when_score_is_none(self, mock_annotation_context): + mock_annotation_context.variant.data = {"score_data": {"score": None}} + + assert _can_annotate_variant_base_assumptions(mock_annotation_context) is False + + def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_annotation_context): + assert _can_annotate_variant_base_assumptions(mock_annotation_context) is True + + +@pytest.mark.unit +class TestPathogenicityAnnotationEligibilityUnit: + def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_annotation_context): + with patch("mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", return_value=False): + result = can_annotate_variant_for_pathogenicity_evidence(mock_annotation_context) + + assert result is False + + def test_pathogenicity_range_check_returns_false_when_no_calibrations_available(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility.calibrations_available_for_annotation", + return_value=[], + ): + result = can_annotate_variant_for_pathogenicity_evidence(mock_annotation_context) + + assert result is False + + def test_pathogenicity_range_check_returns_true_when_all_conditions_met( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set + ) + is True + ) + + def test_a_private_only_calibration_is_ineligible_for_an_anonymous_caller( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set + ) + is False + ) + + def test_a_private_only_calibration_is_eligible_for_an_entitled_caller( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set, principal=admin_principal() + ) + is True + ) + + +@pytest.mark.unit +class TestFunctionalAnnotationEligibilityUnit: + def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_annotation_context) + + assert result is False + + def test_functional_range_check_returns_false_when_no_calibrations_available(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility.calibrations_available_for_annotation", + return_value=[], + ): + result = can_annotate_variant_for_functional_statement(mock_annotation_context) + + assert result is False + + def test_functional_range_check_returns_true_when_all_conditions_met( + self, mock_annotation_context_with_functional_calibration_score_set + ): + assert ( + can_annotate_variant_for_functional_statement(mock_annotation_context_with_functional_calibration_score_set) + is True + ) + + def test_a_private_only_calibration_is_ineligible_for_an_anonymous_caller( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + can_annotate_variant_for_functional_statement(mock_annotation_context_with_functional_calibration_score_set) + is False + ) + + def test_a_private_only_calibration_is_eligible_for_an_entitled_caller( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + can_annotate_variant_for_functional_statement( + mock_annotation_context_with_functional_calibration_score_set, principal=admin_principal() + ) + is True + ) + + +@pytest.mark.integration +class TestAnnotationEligibilityIntegration: + def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): + # Make score presence explicit so a negative result is due to missing calibrations, and give the + # context a real VRS allele to build its subject variant from. + setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} + setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE + context = annotation_context_for(setup_lib_db_with_mapped_variant) + + pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(context) + functional_allowed = can_annotate_variant_for_functional_statement(context) + + # DB fixture score sets do not include calibrations by default, so both should be False. + assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] + assert pathogenicity_allowed is False + assert functional_allowed is False diff --git a/tests/lib/annotation/test_evidence_line.py b/tests/lib/annotation/test_evidence_line.py index fb6f33b74..640b19a5d 100644 --- a/tests/lib/annotation/test_evidence_line.py +++ b/tests/lib/annotation/test_evidence_line.py @@ -21,10 +21,10 @@ from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) -from mavedb.lib.annotation.statement import mapped_variant_to_functional_statement +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit @@ -49,23 +49,23 @@ class TestAcmgEvidenceLine: ) def test_acmg_evidence_line_with_met_valid_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, expected_outcome, expected_strength, expected_direction, ): """Test ACMG evidence line creation with met valid clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) if expected_strength == StrengthOfEvidenceProvided.STRONG: expected_evidence_outcome = expected_outcome.value @@ -73,7 +73,7 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( expected_evidence_outcome = f"{expected_outcome.value}_{expected_strength.name.lower()}" assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Met" @@ -87,11 +87,11 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( def test_acmg_evidence_line_with_not_met_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Test ACMG evidence line creation with not met clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] expected_outcome = VariantPathogenicityEvidenceLine.Criterion.PS3 expected_strength = None expected_evidence_outcome = f"{expected_outcome.value}_not_met" @@ -100,13 +100,13 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Not Met" @@ -117,19 +117,20 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( assert result.targetProposition == proposition assert len(result.hasEvidenceItems) == 1 - def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_mapped_variant): + def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_annotation_context): """Test that ACMG evidence line creation raises error when no calibrations exist.""" - mock_mapped_variant.variant.score_set.score_calibrations = None + context = mock_annotation_context + context.variant.score_set.score_calibrations = None score_calibration = MagicMock() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) - study_result = variant_study_result(mock_mapped_variant) - acmg_evidence_line(mock_mapped_variant, score_calibration, proposition, [study_result]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + acmg_evidence_line(context, score_calibration, proposition, [study_result]) def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_error( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Regression test: VariantPathogenicityEvidenceLine must accept Statement instances directly. @@ -138,22 +139,22 @@ def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_err nested VRS objects from those dicts (e.g. Allele with real genomic coordinates). Evidence model instances must be passed directly, not serialized. """ - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - - study_result = variant_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + + study_result = variant_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, ExperimentalVariantFunctionalImpactClassification.NORMAL, ) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + clinical_proposition = variant_pathogenicity_proposition(context) - result = acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + result = acmg_evidence_line(context, score_calibration, clinical_proposition, [statement]) assert isinstance(result, VariantPathogenicityEvidenceLine) assert len(result.hasEvidenceItems) == 1 @@ -166,16 +167,16 @@ class TestFunctionalEvidenceLine: """Unit tests for functional evidence line creation.""" def test_functional_evidence_line_with_valid_functional_evidence( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test functional evidence line creation with valid evidence.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - study_result = variant_study_result(mapped_variant) - result = functional_evidence_line(mapped_variant, score_calibration, [study_result]) + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + study_result = variant_study_result(context) + result = functional_evidence_line(context, score_calibration, [study_result]) assert isinstance(result, EvidenceLine) - assert result.description == f"Functional evidence line for {mapped_variant.variant.urn}" + assert result.description == f"Functional evidence line for {context.variant.urn}" assert result.directionOfEvidenceProvided is not None assert result.specifiedBy assert result.contributions diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py index 21ecfa173..7480d7e70 100644 --- a/tests/lib/annotation/test_flatten.py +++ b/tests/lib/annotation/test_flatten.py @@ -52,7 +52,7 @@ def test_pathogenicity_calibration_populates_all_fields(self): urn="urn:mavedb:calibration:1", title="Clinical Calibration" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "abnormal" assert annotation.acmg_criterion == "PS3" @@ -65,7 +65,7 @@ def test_pathogenicity_calibration_populates_all_fields(self): def test_benign_criterion_yields_benign_classification(self): calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_criterion == "BS3" assert annotation.pathogenicity_classification == "BENIGN" @@ -80,7 +80,7 @@ def test_functional_only_calibration_omits_acmg_fields(self): [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "normal" assert annotation.acmg_criterion is None @@ -93,7 +93,7 @@ def test_functional_only_calibration_omits_acmg_fields(self): def test_no_calibration_yields_empty_annotation(self): mapped_variant = create_mock_mapped_variant() - assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + assert flatten_annotation(mapped_variant.variant, None) == FlatAnnotation() def test_calibration_without_ranges_keeps_its_identity(self): """Which calibration was consulted is known even when it can classify nothing. @@ -105,7 +105,7 @@ def test_calibration_without_ranges_keeps_its_identity(self): [], urn="urn:mavedb:calibration:3", title="Baseline Only" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.calibration_urn == "urn:mavedb:calibration:3" assert annotation.calibration_title == "Baseline Only" @@ -119,7 +119,7 @@ def test_calibration_without_ranges_keeps_its_identity(self): def test_variant_outside_all_ranges_is_uncertain(self): calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "indeterminate" assert annotation.acmg_criterion == "PS3" @@ -143,7 +143,7 @@ def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code criterion=criterion, evidence_strength=evidence_strength ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_evidence_outcome_code == expected_code @@ -153,7 +153,7 @@ def test_moderate_plus_is_preserved(self): evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_evidence_strength == "MODERATE_PLUS" assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index cb883e0bc..b7c905c60 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -7,51 +7,164 @@ clinical and functional impact propositions. """ +from types import SimpleNamespace +from unittest.mock import patch + import pytest pytest.importorskip("psycopg2") +from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.va_spec.base.core import ( ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition, ) from ga4gh.vrs.models import MolecularVariation +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + sequence_feature_for_variant, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.lib.annotation.conftest import annotation_context_for + + +def _protein_categorical_variant() -> CategoricalVariant: + """A Cat-VRS ``CategoricalVariant`` for a protein-measured variant (defining protein + `encodes` nt). + + Built over transient links (no DB) so the proposition tests can assert the builders accept a + categorical subject — the shape a protein assay produces under Slice 5.1. + """ + links = [ + MappingRecordAllele( + is_authoritative=True, + allele=Allele(level="protein", vrs_digest="prot", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + MappingRecordAllele( + is_authoritative=False, + allele=Allele(level="cdna", vrs_digest="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + return transit.categorical_variant @pytest.mark.unit class TestExperimentalVariantClinicalImpactProposition: """Unit tests for experimental variant clinical impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_clinical_impact_proposition(self, mock_mapped_variant): - """Test creation of clinical impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + def test_variant_pathogenicity_proposition(self, mock_annotation_context): + """Test creation of pathogenicity proposition from a variant annotation context.""" + result = variant_pathogenicity_proposition(mock_annotation_context) assert isinstance(result, VariantPathogenicityProposition) - assert result.description == f"Variant pathogenicity proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant pathogenicity proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "isCausalFor" assert result.objectCondition.root.conceptType == "Disease" assert result.objectCondition.root.primaryCoding.code.root == "C0012634" assert result.objectCondition.root.primaryCoding.system == "https://www.ncbi.nlm.nih.gov/medgen/" + def test_clinical_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_pathogenicity_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + @pytest.mark.unit class TestExperimentalVariantFunctionalImpactProposition: """Unit tests for experimental variant functional impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_functional_impact_proposition(self, mock_mapped_variant): - """Test creation of functional impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_functional_impact_proposition(mock_mapped_variant) + def test_variant_functional_impact_proposition(self, mock_annotation_context): + """Test creation of functional impact proposition from a variant annotation context.""" + result = variant_functional_impact_proposition(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactProposition) - assert result.description == f"Variant functional impact proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "impactsFunctionOf" assert result.objectSequenceFeature.primaryCoding.code.root == "BRCA1" assert result.objectSequenceFeature.primaryCoding.system == "https://www.genenames.org/" assert result.experimentalContextQualifier is not None + + def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_functional_impact_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + + +@pytest.mark.unit +class TestSequenceFeatureForVariantUnit: + """sequence_feature_for_variant — co-located with the propositions that consume it.""" + + def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=None): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) + + def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["ENST00000357654"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "ENST00000357654" + assert system == "https://www.ensembl.org/index.html" + + def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["NM_000546.6"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "NM_000546.6" + assert system == "https://www.ncbi.nlm.nih.gov/refseq/" + + def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["CUSTOM_ID_1"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "CUSTOM_ID_1" + assert system == "transcript or gene identifier of unknown source" + + def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "TP53" + assert system == "https://www.mavedb.org/" + + def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) diff --git a/tests/lib/annotation/test_statement.py b/tests/lib/annotation/test_statement.py index 5c36aeecf..7cd7e41fb 100644 --- a/tests/lib/annotation/test_statement.py +++ b/tests/lib/annotation/test_statement.py @@ -3,7 +3,7 @@ """ Tests for mavedb.lib.annotation.statement module. -This module tests statement creation functions for mapped variants, +This module tests statement creation functions for variants, focusing on functional impact statements and their classification. """ @@ -18,15 +18,13 @@ from mavedb.lib.annotation.annotate import variant_study_result from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import functional_evidence_line -from mavedb.lib.annotation.proposition import mapped_variant_to_experimental_variant_functional_impact_proposition -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, -) +from mavedb.lib.annotation.proposition import variant_functional_impact_proposition +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit -class TestMappedVariantFunctionalStatement: - """Unit tests for mapped variant functional statement creation.""" +class TestFunctionalStatement: + """Unit tests for functional statement creation.""" @pytest.mark.parametrize( "classification, expected_direction", @@ -36,30 +34,26 @@ class TestMappedVariantFunctionalStatement: (ExperimentalVariantFunctionalImpactClassification.INDETERMINATE, Direction.NEUTRAL), ], ) - def test_mapped_variant_to_functional_statement( + def test_functional_statement( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, classification, expected_direction, ): """Test functional statement creation with different classifications.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.functional_classification_of_variant", return_value=(MagicMock(label="Test Range"), classification), ): - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - result = mapped_variant_to_functional_statement( - mapped_variant, proposition, [evidence], score_calibration, classification - ) + proposition = variant_functional_impact_proposition(context) + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + result = functional_statement(context, proposition, [evidence], score_calibration, classification) assert isinstance(result, Statement) - assert result.description == f"Variant functional impact statement for {mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact statement for {context.variant.urn}." assert result.specifiedBy assert result.contributions assert len(result.contributions) == 3 # API, VRS, and score calibration contributions @@ -75,24 +69,22 @@ def test_mapped_variant_to_functional_statement( def test_no_calibrations_raises_value_error( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, ): """Test that missing score calibrations raises ValueError in evidence line creation.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] # Set calibrations to None to trigger the error - mapped_variant.variant.score_set.score_calibrations = None + context.variant.score_set.score_calibrations = None - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + proposition = variant_functional_impact_proposition(context) with pytest.raises(ValueError, match="does not have a score set with score calibrations"): # Use a dummy score calibration for evidence line creation, but it should not be reached due to the error - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - mapped_variant_to_functional_statement( - mapped_variant, + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + functional_statement( + context, proposition, [evidence], score_calibration, @@ -101,8 +93,8 @@ def test_no_calibrations_raises_value_error( @pytest.mark.unit -class TestMappedVariantPathogenicityStatement: - """Unit tests for mapped variant pathogenicity statement ACMG classification mapping.""" +class TestPathogenicityStatement: + """Unit tests for pathogenicity statement ACMG classification mapping.""" def test_pathogenic_criterion_maps_to_pathogenic(self, mock_mapped_variant): """Test that pathogenic ACMG criterion (PS3) maps to PATHOGENIC classification.""" diff --git a/tests/lib/annotation/test_study_result.py b/tests/lib/annotation/test_study_result.py index 1cf8bbf63..356eaa1e2 100644 --- a/tests/lib/annotation/test_study_result.py +++ b/tests/lib/annotation/test_study_result.py @@ -14,24 +14,22 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult from ga4gh.vrs.models import MolecularVariation -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri -from mavedb.lib.annotation.study_result import ( - mapped_variant_to_experimental_variant_impact_study_result, -) +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri +from mavedb.lib.annotation.study_result import variant_impact_study_result @pytest.mark.unit class TestExperimentalVariantImpactStudyResult: """Unit tests for experimental variant impact study result creation.""" - def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_mapped_variant): - """Test creation of experimental variant impact study result from mapped variant.""" - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_variant_impact_study_result(self, mock_annotation_context): + """Test creation of experimental variant impact study result from an annotation context.""" + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -39,18 +37,18 @@ def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_m # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert mapped_variant_as_iri(mock_mapped_variant) in result.reportedIn - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert measured_allele_as_iri(mock_annotation_context.measured_allele) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn - def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): - """Test that study result handles missing mapped variant (no ClinGen allele ID).""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_no_clingen_allele_id_is_filtered_properly(self, mock_annotation_context): + """Test that study result handles a measured allele with no ClinGen allele ID.""" + mock_annotation_context.measured_allele.clingen_allele_id = None + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -58,5 +56,5 @@ def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn assert len(result.reportedIn) == 1 diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py deleted file mode 100644 index d44cbca3f..000000000 --- a/tests/lib/annotation/test_util.py +++ /dev/null @@ -1,900 +0,0 @@ -# ruff: noqa: E402 - -""" -Tests for mavedb.lib.annotation.util module. - -This module tests utility functions used by annotation workflows, including -variation extraction, annotation eligibility checks, and sequence feature -resolution. -""" - -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -pytest.importorskip("psycopg2") -pytest.importorskip("fastapi") - -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.util import ( - _can_annotate_variant_base_assumptions, - calibrations_available_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, - sequence_feature_for_mapped_variant, - variation_from_mapped_variant, - vrs_object_from_mapped_variant, -) -from mavedb.lib.permissions.principal import Principal -from tests.helpers.constants import ( - TEST_SEQUENCE_LOCATION_ACCESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE, - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, -) -from tests.lib.annotation.conftest import admin_principal, make_private - - -def _has_calibrations_for_annotation(*args, **kwargs) -> bool: - """Whether any calibration survived both the eligibility and visibility checks. - - ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below predate - that and assert only on whether the list was empty. - """ - return bool(calibrations_available_for_annotation(*args, **kwargs)) - - -@pytest.mark.unit -class TestVariationExtractionUnit: - @pytest.mark.parametrize( - "variation_version", - [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], - ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], - ) - def test_variation_from_mapped_variant_post_mapped_variation(self, mock_mapped_variant, variation_version): - mock_mapped_variant.post_mapped = variation_version - - result = variation_from_mapped_variant(mock_mapped_variant).model_dump() - - assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION - assert result["location"]["start"] == 5 - assert result["location"]["end"] == 6 - - def test_variation_from_mapped_variant_no_post_mapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - with pytest.raises(MappingDataDoesntExistException): - variation_from_mapped_variant(mock_mapped_variant) - - @pytest.mark.parametrize( - "allele_dict, expected_state_type, expected_state_fields", - [ - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE, - "LiteralSequenceExpression", - {"sequence": "F"}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, - "ReferenceLengthExpression", - {"length": 5, "repeatSubunitLength": 5}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - "LengthExpression", - {"length": 5}, - ), - ], - ids=["lse_state", "rle_state", "length_expression_state"], - ) - def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): - result = vrs_object_from_mapped_variant(allele_dict).model_dump() - - assert result["state"]["type"] == expected_state_type - for k, v in expected_state_fields.items(): - assert result["state"][k] == v - - def test_unknown_allele_state_type_raises_value_error(self): - allele_dict = { - **TEST_VALID_POST_MAPPED_VRS_ALLELE, - "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, - } - - with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): - vrs_object_from_mapped_variant(allele_dict) - - @pytest.mark.parametrize( - "member_dict, expected_state_type", - [ - (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), - ], - ids=["lse_members", "rle_members", "length_expression_members"], - ) - def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): - mapping_results = {"type": "CisPhasedBlock", "members": [member_dict, member_dict]} - - result = vrs_object_from_mapped_variant(mapping_results).model_dump() - - assert result["type"] == "CisPhasedBlock" - assert len(result["members"]) == 2 - assert result["members"][0]["state"]["type"] == expected_state_type - - -@pytest.mark.unit -class TestBaseAnnotationAssumptionsUnit: - def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): - mock_mapped_variant.variant.data = {"score_data": {"score": None}} - - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is False - - def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is True - - -@pytest.mark.unit -class TestScoreCalibrationMayBeUsedForAnnotation: - def test_returns_false_for_research_use_only_when_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=False, - ) - is False - ) - - def test_returns_true_for_research_use_only_when_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=True, - ) - is True - ) - - def test_returns_false_when_functional_classifications_missing( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.functional_classifications = [] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False - - def test_returns_false_for_pathogenicity_without_acmg_classifications( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for functional_classification in acmg_removed: - functional_classification["acmgClassification"] = None - calibration.functional_classifications = acmg_removed - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False - - def test_returns_true_for_pathogenicity_with_any_acmg_classification( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True - - -@pytest.mark.unit -class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: - """ - Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. - This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. - """ - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = None - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = [] - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ): - """Test that research use only calibrations are excluded by default.""" - # Make all calibrations research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: - calibration.research_use_only = True - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, annotation_type) - is False - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( - self, kind, variant_fixture, request - ): - """Test that research use only calibrations are included when explicitly allowed.""" - mock_mapped_variant = request.getfixturevalue(variant_fixture) - # Make all calibrations research use only - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.primary = False - calibration.research_use_only = True - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant, kind, allow_research_use_only_calibrations=True) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.functional_classifications = None - - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - for fr in acmg_classification_removed: - fr["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - acmg_classification_removed[0]["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is True - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for functional annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - def test_score_range_check_handles_mixed_functional_classifications( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test behavior when some calibrations have functional classifications and some don't.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # First calibration has functional classifications (should already exist) - # Second calibration has no functional classifications - calibrations[1].functional_classifications = None - - # Should return True because at least one calibration has valid functional classifications - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is True - ) - - def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Remove ACMG classifications from all functional classifications - for calibration in calibrations: - if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: - acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for fc in acmg_classification_removed: - if "acmgClassification" in fc: - fc["acmgClassification"] = None - calibration.functional_classifications = acmg_classification_removed - - # Should return False because no ACMG classifications exist - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity") - is False - ) - - def test_functional_annotation_with_empty_functional_classifications_list( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that functional annotation fails when functional classifications list is empty.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Set functional classifications to empty list - for calibration in calibrations: - calibration.functional_classifications = [] - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is False - ) - - -@pytest.mark.unit -class TestCalibrationAvailabilityIsScopedToTheCaller: - """A calibration's READ rule is stricter than its score set's, so publishing a score set does not - publish its calibrations. These cases exist because this function once read - ``score_set.score_calibrations`` directly and handed private calibrations to anyone. - """ - - def test_a_private_calibration_is_not_available_for_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation(mapped_variant, "functional") == [] - - def test_omitting_the_principal_withholds_rather_than_widens( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - # The whole design rests on an omitted principal meaning "the public". Passing an explicitly - # anonymous principal and passing none at all must agree. - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation( - mapped_variant, "functional" - ) == calibrations_available_for_annotation(mapped_variant, "functional", principal=Principal()) - - def test_an_entitled_caller_still_receives_a_private_calibration( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation(mapped_variant, "functional", principal=admin_principal()) != [] - - def test_a_public_calibration_is_still_available_to_anyone( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - # The counterweight: withholding private calibrations must not withhold what publishing released. - assert ( - calibrations_available_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - != [] - ) - - -@pytest.mark.unit -class TestPathogenicityAnnotationEligibilityUnit: - def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", return_value=False): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util.calibrations_available_for_annotation", - return_value=[], - ): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - assert ( - can_annotate_variant_for_pathogenicity_evidence( - mock_mapped_variant_with_pathogenicity_calibration_score_set - ) - is True - ) - - -@pytest.mark.unit -class TestFunctionalAnnotationEligibilityUnit: - def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_false_when_functional_classifications_check_fails( - self, mock_mapped_variant - ): - with patch( - "mavedb.lib.annotation.util.calibrations_available_for_annotation", - return_value=[], - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - assert ( - can_annotate_variant_for_functional_statement(mock_mapped_variant_with_functional_calibration_score_set) - is True - ) - - -@pytest.mark.unit -class TestSequenceFeatureForMappedVariantUnit: - def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=None): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["ENST00000357654"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "ENST00000357654" - assert system == "https://www.ensembl.org/index.html" - - def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["NM_000546.6"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "NM_000546.6" - assert system == "https://www.ncbi.nlm.nih.gov/refseq/" - - def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["CUSTOM_ID_1"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "CUSTOM_ID_1" - assert system == "transcript or gene identifier of unknown source" - - def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "TP53" - assert system == "https://www.mavedb.org/" - - def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - -@pytest.mark.integration -class TestAnnotationUtilIntegration: - """Integration tests for utility helpers using persisted DB-backed variants.""" - - def test_variation_from_persisted_mapped_variant(self, setup_lib_db_with_mapped_variant): - setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE - variation = variation_from_mapped_variant(setup_lib_db_with_mapped_variant) - - assert variation is not None - assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} - - def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): - # Make score presence explicit so a negative result is due to missing calibrations. - setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} - - pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(setup_lib_db_with_mapped_variant) - functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant) - - # DB fixture score sets do not include calibrations by default, so both should be False. - assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] - assert pathogenicity_allowed is False - assert functional_allowed is False - - -@pytest.mark.unit -class TestSelectStrongestFunctionalCalibrationUnit: - """Unit tests for select_strongest_functional_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that first calibration is returned when all have same classification.""" - from copy import deepcopy - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - # Create multiple calibrations with same classification - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] # Should return the first one - - def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that normal classification is preferred when there are conflicts.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different classifications - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - side_effect=[ - (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), - (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), - ], - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return the normal classification (second one) - assert calibration == calibration2 - assert functional_range.label == "Normal" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range but INDETERMINATE classification (variant not in any range) - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None - - -@pytest.mark.unit -class TestSelectStrongestPathogenicityCalibrationUnit: - """Unit tests for select_strongest_pathogenicity_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that calibration with strongest evidence is selected.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different evidence strengths - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ( - MagicMock(label="Strong"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.VERY_STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with VERY_STRONG evidence - assert calibration == calibration2 - assert functional_range.label == "Strong" - - def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength but different criteria (pathogenic vs benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Pathogenic"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration but None for range to indicate uncertain significance - assert calibration == calibration1 - assert functional_range is None - - def test_returns_classification_when_all_tied_are_same_type( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that classification is returned normally when all tied candidates are the same type.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength and same type of criteria (both benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Benign1"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign2"), - VariantPathogenicityEvidenceLine.Criterion.BP1, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with its range (no conflict, so normal classification) - assert calibration == calibration1 - assert functional_range.label == "Benign1" - - def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that None evidence strength is handled correctly.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock with None and actual strength - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with actual strength (second) - assert calibration == calibration2 - assert functional_range.label == "Moderate" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range for all calibrations (variant not in any range) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py new file mode 100644 index 000000000..440646244 --- /dev/null +++ b/tests/lib/clingen/test_alleles.py @@ -0,0 +1,53 @@ +"""Unit tests for the pure allele-grouping primitive shared by the annotation jobs.""" + +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation + + +def _row(allele_id, variant_id, *, caid="CA1"): + return ScoreSetAlleleRow( + allele_id=allele_id, + post_mapped={"type": "Allele"}, + clingen_allele_id=caid, + variant_id=variant_id, + ) + + +def test_collapses_rows_to_one_payload_per_allele(): + rows = [ + _row(1, 10), + _row(1, 11), + _row(2, 12, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {1: "CA1", 2: "CA2"} + + +def test_shared_allele_yields_a_single_entry(): + """An allele linked by multiple variants is deduped to one work-unit (events are allele-keyed).""" + rows = [ + _row(1, 10), + _row(1, 11), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {1: "CA1"} + + +def test_payload_returning_none_skips_the_allele(): + """Returning None from payload drops the allele entirely — replacing each job's ad-hoc + 'no CAID / no HGVS -> continue' filter.""" + rows = [ + _row(1, 10, caid=None), + _row(2, 11, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {2: "CA2"} + + +def test_empty_rows_yield_empty_grouping(): + assert group_alleles_for_annotation([], payload=lambda r: r.clingen_allele_id) == {} diff --git a/tests/lib/clingen/test_content_constructors.py b/tests/lib/clingen/test_content_constructors.py index 7aab47f76..f691a4497 100644 --- a/tests/lib/clingen/test_content_constructors.py +++ b/tests/lib/clingen/test_content_constructors.py @@ -11,7 +11,6 @@ ) from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE from mavedb import __version__ -import pytest from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -54,10 +53,8 @@ def test_construct_ldh_submission_event(): } -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None - result = construct_ldh_submission_entity(mock_variant, mapped_variant) +def test_construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele): + result = construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele) assert "MaveDBMapping" in result assert len(result["MaveDBMapping"]) == 1 @@ -65,15 +62,9 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ assert mapping["entContent"]["mavedb_id"] == VALID_VARIANT_URN assert mapping["entContent"]["score"] == 1.0 - - if has_mapped_variant: - assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" - else: - assert "pre_mapped" not in mapping["entContent"] - assert "post_mapped" not in mapping["entContent"] - assert "mapping_api_version" not in mapping["entContent"] + assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" assert mapping["entId"] == VALID_VARIANT_URN assert ( @@ -82,12 +73,10 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ ) -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None +def test_construct_ldh_submission(mock_variant, mock_mapping_record, mock_allele): variant_content = [ - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), ] uuid_1 = UUID("12345678-1234-5678-1234-567812345678") diff --git a/tests/lib/clingen/test_services.py b/tests/lib/clingen/test_services.py index f0a4f5103..316a6a1a4 100644 --- a/tests/lib/clingen/test_services.py +++ b/tests/lib/clingen/test_services.py @@ -11,7 +11,11 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import ( + CLINGEN_HTTP_TIMEOUT, + GENBOREE_ACCOUNT_NAME, + GENBOREE_ACCOUNT_PASSWORD, +) from mavedb.lib.clingen.services import ( ClinGenAlleleRegistryService, ClinGenLdhService, @@ -61,6 +65,7 @@ def test_authenticate_with_new_jwt(self, mock_existing_jwt, mock_post, clingen_s mock_post.assert_called_once_with( f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth", json={"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.post") @@ -150,6 +155,7 @@ def test_dispatch_submissions_success(self, mock_batched, mock_authenticate, moc url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.put") @@ -169,6 +175,7 @@ def test_dispatch_submissions_failure(self, mock_authenticate, mock_request, cli url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.put") @@ -211,6 +218,7 @@ def test_dispatch_submissions_no_batching(self, mock_batched, mock_authenticate, url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py index 2ac2183fb..a6e28a698 100644 --- a/tests/lib/conftest.py +++ b/tests/lib/conftest.py @@ -135,6 +135,9 @@ def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant): """ Sets up the lib test db with a user, reference, license, and a score set. + + Seeds only the frozen ``MappedVariant`` table; tests that need a mapping on the allele substrate seed + their own record, so that they control its alleles. ``tests/lib/csv`` overrides this to add one. """ mapped_variant = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=setup_lib_db_with_variant.id) @@ -307,6 +310,23 @@ def mock_mapped_variant(mock_variant): return mv +@pytest.fixture +def mock_mapping_record(mock_mapped_variant): + # Pre-mapped data and the mapping API version now live on the per-variant MappingRecord. + rec = mock.Mock() + rec.pre_mapped = mock_mapped_variant.pre_mapped + rec.mapping_api_version = mock_mapped_variant.mapping_api_version + return rec + + +@pytest.fixture +def mock_allele(mock_mapped_variant): + # Post-mapped data now lives on the (cross-variant deduped) Allele. + allele = mock.Mock() + allele.post_mapped = mock_mapped_variant.post_mapped + return allele + + @pytest.fixture def mock_mapped_variant_with_functional_calibration_score_set( mock_mapped_variant, mock_variant_with_functional_calibration_score_set diff --git a/tests/lib/csv/conftest.py b/tests/lib/csv/conftest.py new file mode 100644 index 000000000..f79741225 --- /dev/null +++ b/tests/lib/csv/conftest.py @@ -0,0 +1,45 @@ +# ruff: noqa: E402 + +"""Fixtures scoped to the CSV library tests.""" + +import pytest + +pytest.importorskip("psycopg2") + +from tests.helpers.constants import TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +@pytest.fixture +def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant, setup_lib_db_with_mapped_variant): + """Give the shared fixture's variant a live mapping on the allele substrate as well. + + The CSV surfaces resolve their mapping-derived namespaces from the allele graph, not from the frozen + ``MappedVariant`` row the parent fixture seeds, so without this every such column comes back NA. + Overridden here rather than in the parent conftest because tests elsewhere under ``tests/lib`` seed + their own records, and a second live record per variant violates ``uq_mapping_records_current``. + + Seeded as the full g/c/p triple the CSV reconstructs from — an authoritative genomic allele, its + projection-group cdna sibling, and the protein apex — so a test can populate any level's HGVS by + setting it on that level's allele. Mirrors ``tests.helpers.util.score_set.seed_csv_substrate``. + """ + seed_mapping_record( + session, + setup_lib_db_with_variant, + assay_level="genomic", + hgvs_assay_level=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + alleles=[ + AlleleSpec( + digest="lib-csv-genomic", + level="genomic", + is_authoritative=True, + projection_group=0, + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["vep_functional_consequence"], + ), + AlleleSpec(digest="lib-csv-cdna", level="cdna", projection_group=0), + AlleleSpec(digest="lib-csv-protein", level="protein"), + ], + ) + session.refresh(setup_lib_db_with_mapped_variant) + return setup_lib_db_with_mapped_variant diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index c0d97fa68..595a60343 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -16,7 +16,7 @@ from mavedb.lib.csv.namespaces import CsvNamespace from tests.helpers.constants import VALID_CALIBRATION_URN from tests.helpers.mocks.factories import create_mock_mapped_variant -from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids, to_csv_mapped_row # --------------------------------------------------------------------------- # MockVariant @@ -617,7 +617,9 @@ def test_a_row_composes_and_carries_every_planned_column(self, shape): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) assert set(row) == set(assemble_csv_headers(plan.namespaced_columns)) @@ -634,7 +636,9 @@ def test_no_cell_leaks_a_mock(self, shape): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) leaked = {column: value for column, value in row.items() if "Mock" in str(value)} assert not leaked, f"{shape.name} leaked mock reprs into the row: {leaked}" @@ -645,7 +649,9 @@ def test_a_mapping_without_hgvs_columns_renders_them_na(self): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) assert row["post_mapped_hgvs_c"] == "NA" assert row["post_mapped_hgvs_at_assay_level"] == "NA" @@ -658,7 +664,9 @@ def test_the_row_serializes_to_csv(self, shape): plan = self._plan() columns = assemble_csv_headers(plan.namespaced_columns) - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) rendered = rows_to_csv([row], columns) assert len(list(csv.reader(StringIO(rendered)))) == 2 diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 6b475d6fd..74f5ecf66 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -2,8 +2,12 @@ import csv import io -from datetime import date -from unittest.mock import Mock, patch +from unittest.mock import Mock + +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from unittest.mock import patch import pytest @@ -21,12 +25,17 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.acmg_classification import ACMGClassification -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.enums.user_role import UserRole from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet @@ -160,21 +169,89 @@ def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_a db.commit() db.refresh(mapped_variant) + # Equivalence is resolved on the allele substrate: two measurements are the same variant when their + # authoritative alleles carry the same ClinGen allele id. + seed_mapping_record( + db, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest=f"equivalent-{variant.id}", + level="genomic", + is_authoritative=True, + clingen_allele_id=clingen_allele_id, + ) + ], + ) + db.refresh(mapped_variant) + return score_set, variant, mapped_variant def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): - mapped_variant.clinical_controls.append( - ClinicalControl( - db_identifier="183058", - gene_symbol="PTEN", - clinical_significance=significance, - clinical_review_status=review_status, - db_name="ClinVar", - db_version=db_version, - ) + """Link a control to the variant's authoritative allele, which is where the CSV layer reads it.""" + control = ClinvarControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, ) - db.add(mapped_variant) + db.add(control) + db.commit() + + allele = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == mapped_variant.variant_id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, "no authoritative allele to link a ClinVar control to" + + db.add(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id)) + db.commit() + + +def _live_alleles_by_level(db, variant_id): + """The live mapping record's alleles, keyed by level — what the CSV actually reads.""" + alleles = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.valid_to.is_(None)) + ).all() + return {allele.level: allele for allele in alleles} + + +def _live_mapping_record(db, variant_id): + record = db.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant_id).where(MappingRecord.valid_to.is_(None)) + ).one() + return record + + +def _authoritative_allele(db, variant_id): + allele = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, f"variant {variant_id} has no authoritative allele" + return allele + + +def _link_gnomad(db, allele, gnomad_variant): + db.add(gnomad_variant) + db.commit() + db.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) db.commit() @@ -346,8 +423,7 @@ def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_m def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -367,8 +443,7 @@ def test_each_measurement_is_interpreted_only_under_its_own_calibrations( ): """A score from one assay carries no meaning under another assay's thresholds.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -399,8 +474,7 @@ def test_each_measurement_is_interpreted_only_under_its_own_calibrations( def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): """A calibration means nothing against another score set's scores, so say which one owns it.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -449,20 +523,21 @@ def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" - mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" - mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" - mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - mapped_variant.vep_functional_consequence = "missense_variant" - mapped_variant.clingen_allele_id = "CA123456" - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) + # The triple reconstructs from three different places, not from one row's columns: the measured + # slot is the record's assay-level string, the other nucleotide slot is the projection-group + # partner's HGVS, and the protein slot is the apex allele's. + by_level = _live_alleles_by_level(session, mapped_variant.variant_id) + record = _live_mapping_record(session, mapped_variant.variant_id) + record.hgvs_assay_level = "NC_000010.11:g.87933147C>T" + by_level["cdna"].hgvs_c = "NM_000314.8:c.100A>G" + by_level["protein"].hgvs_p = "NP_000305.3:p.Lys34Glu" + authoritative = by_level["genomic"] + authoritative.vrs_digest = TEST_GA4GH_IDENTIFIER + authoritative.clingen_allele_id = "CA123456" session.commit() + _link_gnomad(session, authoritative, GnomADVariant(**TEST_GNOMAD_VARIANT)) - # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` - # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" @@ -473,33 +548,38 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) assert rows[0]["clingen.clingen_allele_id"] == "CA123456" - def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setup_lib_db_with_mapped_variant): - """A post-mapped object carrying only a ``digest`` reports NA rather than a built-up CURIE. + def test_post_mapped_vrs_id_is_the_stored_identifier_verbatim(self, session, setup_lib_db_with_mapped_variant): + """The column is reported as-is, never rebuilt from the post-mapped payload. - The two stored fields are known to disagree on some rows, and only ``id`` is indexed and matched - by the VRS lookup, so a digest-derived identifier would resolve to nothing. + The old resolver parsed the stored VRS object and could fall back to synthesizing a CURIE from its + ``digest`` — which resolved to nothing, because the two fields are known to disagree on some rows + and only ``id`` is indexed. The allele substrate removes the fallback entirely: ``Allele.vrs_digest`` + already holds the full prefixed identifier, so there is nothing to derive and no payload to parse. """ mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.post_mapped = { + allele = _authoritative_allele(session, mapped_variant.variant_id) + allele.vrs_digest = TEST_GA4GH_IDENTIFIER + # A payload disagreeing with the column must not influence the reported identifier. + allele.post_mapped = { key: value for key, value in TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X.items() if key != "id" } - session.add(mapped_variant) session.commit() rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) - assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + assert rows[0]["mavedb.post_mapped_vrs_id"] == TEST_GA4GH_IDENTIFIER + assert rows[0]["mavedb.post_mapped_vrs_id"] != TEST_GA4GH_DIGEST def test_gnomad_namespace_reports_the_whole_frequency_record(self, session, setup_lib_db_with_mapped_variant): """AF alone cannot be linked out from or judged for sampling depth; the namespace carries the record.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) - session.commit() + _link_gnomad( + session, + _authoritative_allele(session, mapped_variant.variant_id), + GnomADVariant(**TEST_GNOMAD_VARIANT), + ) - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - csv_text = get_variant_csv(session, mapped_variant.variant.urn) - rows = _parse_csv(csv_text) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert [column for column in rows[0].keys() if column.startswith("gnomad.")] == [ "gnomad.gnomad_af", @@ -522,26 +602,32 @@ def test_gnomad_record_absent_leaves_every_column_na(self, session, setup_lib_db """A variant with no gnomAD record reports NA across the namespace, never a zero frequency.""" mapped_variant = setup_lib_db_with_mapped_variant - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) gnomad_values = {key: value for key, value in rows[0].items() if key.startswith("gnomad.")} assert len(gnomad_values) == 7 assert set(gnomad_values.values()) == {"NA"} - def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + def test_a_link_to_an_older_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + """The live link decides the release, not the release the deployment currently serves. + + The linker only visits alleles present in the release it ingests, so an allele absent from a newer + release keeps its prior-release link live -- indefinitely, not just during re-ingest. Filtering the + read on the served-release constant turned every such allele's frequency into a permanent NA. The + honest answer is the value we hold, labeled with the release it came from. + """ mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) - session.commit() + _link_gnomad( + session, + _authoritative_allele(session, mapped_variant.variant_id), + GnomADVariant(**{**TEST_GNOMAD_VARIANT, "db_version": "v2.1.1"}), + ) - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) - # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD - # record from another version leaves the frequency NA rather than dropping the variant. assert len(rows) == 1 - assert rows[0]["gnomad.gnomad_af"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["gnomad.gnomad_version"] == "v2.1.1" def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant @@ -560,7 +646,7 @@ def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant mapped_variant.clinical_controls.append( - ClinicalControl( + ClinvarControl( db_identifier="ABC123", gene_symbol="BRCA1", clinical_significance="benign", @@ -615,8 +701,15 @@ def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( ) session.add(mapped_sibling) session.commit() - session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) - session.commit() + # The sibling carries the score set's mapping; the requested variant has none, which is what + # makes NA the honest value rather than an absent column. + seed_mapping_record( + session, + mapped_sibling, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.1A>G", + alleles=[AlleleSpec(digest="sibling-genomic", level="genomic", is_authoritative=True)], + ) rows = _parse_csv(get_variant_csv(session, variant.urn)) @@ -635,18 +728,26 @@ def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib assert "scores.score" in csv_text.splitlines()[0] def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + """Re-mapping retires the prior record rather than clearing a flag, and only the live one is read.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.current = False - session.add(mapped_variant) - session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, - variant_id=mapped_variant.variant_id, - clingen_allele_id="CA999999", - ) - ) + _live_mapping_record(session, mapped_variant.variant_id).retire(session) session.commit() + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.2A>G", + alleles=[ + AlleleSpec( + digest="superseding-genomic", + level="genomic", + is_authoritative=True, + clingen_allele_id="CA999999", + ) + ], + ) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert len(rows) == 1 @@ -925,84 +1026,115 @@ def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mappe # --------------------------------------------------------------------------- -class TestAnchorMappingIsDeterministic: - """Everything in the CSV follows from which current mapping anchors the request.""" +class TestOneLiveMappingPerVariantIsGuaranteed: + """The CSV no longer has to tie-break between mappings claiming to be current. - def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( - self, session, setup_lib_db_with_mapped_variant - ): - """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + The old substrate could not stop two ``MappedVariant`` rows from both carrying ``current=True``, so + the exports pinned a chosen row id to keep repeat downloads agreeing. The allele graph promotes that + invariant to the database -- ``uq_mapping_records_current`` is unique on ``variant_id`` where + ``valid_to IS NULL`` -- so the ambiguity those tests guarded against is now unrepresentable, and the + pinning they justified has been removed. What is worth asserting is the guarantee itself. + """ + + def test_a_second_live_mapping_record_is_rejected(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant + # The fixture already seeded one live record for this variant. + session.add(MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0")) - newer = MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, - variant_id=variant.id, - ) - session.add(newer) - session.commit() + with pytest.raises(IntegrityError): + session.commit() + session.rollback() - first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) - second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + def test_a_superseded_record_does_not_block_a_new_live_one(self, session, setup_lib_db_with_mapped_variant): + """Supersession is what makes re-mapping possible: the prior record closes, the new one opens.""" + variant = setup_lib_db_with_mapped_variant.variant + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() - assert first == second - # The requested variant anchors the export and comes first, so this pins which mapping was - # picked rather than merely whether the newer one appears anywhere in the output. - assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" + live.retire(session) + session.add(MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.1.0")) + session.commit() - def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): - """Two current mappings on one variant are the same measurement twice, not two equivalents. + still_live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).all() + assert len(still_live) == 1 + assert still_live[0].mapping_api_version == "test.1.0" - Emitting both would also break the row ordering downstream, which restores the caller's order from - the variant ids alone. - """ - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) - session.commit() - variant = mapped_variant.variant - session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, - variant_id=variant.id, - ) - ) - session.commit() +# --------------------------------------------------------------------------- +# TestScoreSetCsvCalibrationColumns +# --------------------------------------------------------------------------- - # A genuine equivalent in another score set, so the widening is doing something to dedupe within. - _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") - rows = _parse_csv(get_variant_csv(session, variant.urn)) +@pytest.mark.integration +class TestGnomadReleasesCoexistOnTheReadPath: + """A score set can hold live links at more than one gnomAD release, and the read path must serve both. - assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + gnomAD shadows like VEP rather than accumulating like ClinVar: the link is allele-keyed and + single-live, so a release bump supersedes an allele's prior link. But ingestion only visits alleles + present in the release being ingested, so alleles absent from a newer one keep their older link live. + A corpus therefore sits at mixed releases as its steady state, and one export has to carry both. + """ - def test_an_equivalent_variants_extra_current_mapping_is_reported_once( - self, session, setup_lib_db_with_mapped_variant - ): - """The same rule applies to the widened rows, not just to the anchor.""" - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + def _variant_at(self, session, score_set, urn_suffix, digest, db_version, allele_frequency): + variant = Variant(**{**TEST_MINIMAL_VARIANT, "urn": f"{score_set.urn}#{urn_suffix}"}, score_set_id=score_set.id) + session.add(variant) session.commit() - - variant = mapped_variant.variant - _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") - session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, - variant_id=other_variant.id, - ) + seed_mapping_record( + session, + variant, + assay_level="genomic", + hgvs_assay_level=f"NC_000010.11:g.{urn_suffix}A>G", + alleles=[AlleleSpec(digest=digest, level="genomic", is_authoritative=True)], ) - session.commit() - - rows = _parse_csv(get_variant_csv(session, variant.urn)) + _link_gnomad( + session, + _authoritative_allele(session, variant.id), + GnomADVariant( + **{ + **TEST_GNOMAD_VARIANT, + "db_version": db_version, + "db_identifier": f"10-{urn_suffix}-A-G", + "allele_frequency": allele_frequency, + } + ), + ) + return variant - assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + def test_two_releases_are_reported_side_by_side_each_with_its_own_version(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + newer = self._variant_at(session, score_set, 2, "gnomad-newer", "v4.1", 0.25) + older = self._variant_at(session, score_set, 3, "gnomad-older", "v2.1.1", 0.5) + + rows = _parse_csv(get_score_set_variants_as_csv(session, score_set, ["scores", "gnomad"], namespaced=True)) + + by_urn = {row["accession"]: row for row in rows} + # Both frequencies are served, each labeled with the release it actually came from. + assert by_urn[newer.urn]["gnomad.gnomad_af"] == "0.25" + assert by_urn[newer.urn]["gnomad.gnomad_version"] == "v4.1" + assert by_urn[older.urn]["gnomad.gnomad_af"] == "0.5" + assert by_urn[older.urn]["gnomad.gnomad_version"] == "v2.1.1" + # The variant with no gnomAD link at all is still a row, with the namespace NA. + assert by_urn[setup_lib_db_with_variant.urn]["gnomad.gnomad_af"] == "NA" + assert by_urn[setup_lib_db_with_variant.urn]["gnomad.gnomad_version"] == "NA" + + def test_neither_release_is_dropped_by_the_served_release_constant(self, session, setup_lib_db_with_variant): + """Whatever `GNOMAD_DATA_VERSION` the deployment serves, the read path does not consult it. + + Ingestion still needs the constant -- it names the release being ingested and derives the source + table -- so this asserts only that reads go off live rows. + """ + score_set = setup_lib_db_with_variant.score_set + self._variant_at(session, score_set, 2, "gnomad-a", "v4.1", 0.25) + self._variant_at(session, score_set, 3, "gnomad-b", "v2.1.1", 0.5) + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v99.99"): + rows = _parse_csv(get_score_set_variants_as_csv(session, score_set, ["gnomad"], namespaced=True)) -# --------------------------------------------------------------------------- -# TestScoreSetCsvCalibrationColumns -# --------------------------------------------------------------------------- + reported = {row["gnomad.gnomad_version"] for row in rows} + assert reported == {"v4.1", "v2.1.1", "NA"} class TestScoreSetCsvCalibrationColumns: diff --git a/tests/lib/permissions/test_score_calibration.py b/tests/lib/permissions/test_score_calibration.py index a9ea8370d..2eede0fb6 100644 --- a/tests/lib/permissions/test_score_calibration.py +++ b/tests/lib/permissions/test_score_calibration.py @@ -178,7 +178,9 @@ class TestScoreCalibrationReadActionHandler: "ScoreCalibration", "private", "anonymous", Action.READ, False, 404, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_read_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_read_action helper function directly.""" @@ -265,7 +267,9 @@ class TestScoreCalibrationUpdateActionHandler: "ScoreCalibration", "published", "anonymous", Action.UPDATE, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_update_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_update_action helper function directly.""" @@ -350,7 +354,9 @@ class TestScoreCalibrationDeleteActionHandler: "ScoreCalibration", "published", "anonymous", Action.DELETE, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_delete_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_delete_action helper function directly.""" @@ -431,7 +437,9 @@ class TestScoreCalibrationPublishActionHandler: "ScoreCalibration", "published", "anonymous", Action.PUBLISH, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_publish_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_publish_action helper function directly.""" @@ -561,7 +569,9 @@ class TestScoreCalibrationChangeRankActionHandler: investigator_provided=False, ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_change_rank_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_change_rank_action helper function directly.""" diff --git a/tests/lib/test_allele_annotations.py b/tests/lib/test_allele_annotations.py new file mode 100644 index 000000000..cedb4267c --- /dev/null +++ b/tests/lib/test_allele_annotations.py @@ -0,0 +1,162 @@ +# ruff: noqa: E402 +"""Integration tests for the digest-keyed annotation assembler (``lib/allele_annotations.py``). + +These pin: the map is keyed by ``vrs_digest`` and covers every allele handed in (an allele with no +annotations still gets an empty entry); VEP/gnomAD resolve to one value each while ClinVar is a list +(multi-live, one per release); absence is ``None``/empty; and ``as_of`` reconstructs each source at a +past instant over the immutable, content-addressed alleles. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_annotations import get_allele_annotations +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="cdna"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _vep(session, allele, value, *, source_version="116", valid_from=None): + row = VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=value, + source_version=source_version, + access_date="2026-01-01", + valid_from=valid_from, + ) + session.add(row) + session.commit() + return row + + +def _gnomad(session, allele, *, allele_frequency=0.0123, db_version="4.1.0"): + gv = GnomADVariant( + db_name="gnomAD", + db_identifier=f"gnomad-{allele.vrs_digest}", + db_version=db_version, + allele_count=42, + allele_number=10000, + allele_frequency=allele_frequency, + faf95_max=0.05, + ) + session.add(gv) + session.commit() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gv.id) + session.add(link) + session.commit() + return gv + + +def _clinvar(session, allele, *, significance="Likely benign", db_version="11_2024", variation_id="12345"): + control = ClinvarControl( + db_identifier=f"cv-{db_version}-{allele.vrs_digest}", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status="criteria provided, multiple submitters, no conflicts", + db_name="ClinVar", + db_version=db_version, + clinvar_variation_id=variation_id, + ) + session.add(control) + session.commit() + link = ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id) + session.add(link) + session.commit() + return control + + +@pytest.mark.integration +def test_empty_allele_list_returns_empty_map(session, setup_lib_db): + assert get_allele_annotations(session, []) == {} + + +@pytest.mark.integration +def test_allele_without_annotations_gets_an_empty_entry(session, setup_lib_db): + """The map's keys mirror the alleles handed in — a fully un-annotated allele still gets an + (empty) block, so the envelope can join every Cat-VRS member to something.""" + allele = _allele(session, "bare-digest") + + annotations = get_allele_annotations(session, [allele]) + + assert set(annotations) == {"bare-digest"} + block = annotations["bare-digest"] + assert block.vep is None + assert block.gnomad is None + assert block.clinvar == [] + + +@pytest.mark.integration +def test_all_three_sources_present(session, setup_lib_db): + allele = _allele(session, "rich-digest") + _vep(session, allele, "missense_variant") + _gnomad(session, allele, allele_frequency=0.0123) + _clinvar(session, allele, significance="Likely benign") + + block = get_allele_annotations(session, [allele])["rich-digest"] + + assert block.vep is not None and block.vep.consequence == "missense_variant" + assert block.vep.source_version == "116" + assert block.gnomad is not None and block.gnomad.allele_frequency == 0.0123 + assert block.gnomad.faf95_max == 0.05 + assert [c.clinical_significance for c in block.clinvar] == ["Likely benign"] + + +@pytest.mark.integration +def test_clinvar_is_multi_live_across_releases(session, setup_lib_db): + """ClinVar stacks one live link per release, so an allele can carry several assertions at once.""" + allele = _allele(session, "cv-digest") + _clinvar(session, allele, significance="Likely benign", db_version="11_2024") + _clinvar(session, allele, significance="Pathogenic", db_version="05_2025") + + block = get_allele_annotations(session, [allele])["cv-digest"] + + assert {(c.db_version, c.clinical_significance) for c in block.clinvar} == { + ("11_2024", "Likely benign"), + ("05_2025", "Pathogenic"), + } + + +@pytest.mark.integration +def test_map_is_keyed_by_digest_across_multiple_alleles(session, setup_lib_db): + a1 = _allele(session, "digest-1", level="cdna") + a2 = _allele(session, "digest-2", level="protein") + _vep(session, a1, "missense_variant") + _gnomad(session, a2) + + annotations = get_allele_annotations(session, [a1, a2]) + + assert set(annotations) == {"digest-1", "digest-2"} + assert annotations["digest-1"].vep is not None and annotations["digest-1"].gnomad is None + assert annotations["digest-2"].gnomad is not None and annotations["digest-2"].vep is None + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation(session, setup_lib_db): + """A changed VEP consequence supersedes the old row; as_of selects the value live at the instant.""" + allele = _allele(session, "vep-digest") + old = _vep(session, allele, "missense_variant", source_version="110", valid_from=T0) + old.retire(session, at=T1) + session.commit() + _vep(session, allele, "stop_gained", source_version="116", valid_from=T1) + + current = get_allele_annotations(session, [allele])["vep-digest"] + historical = get_allele_annotations(session, [allele], as_of=T1 - timedelta(days=1))["vep-digest"] + + assert current.vep is not None and current.vep.consequence == "stop_gained" + assert historical.vep is not None and historical.vep.consequence == "missense_variant" diff --git a/tests/lib/test_allele_detail.py b/tests/lib/test_allele_detail.py new file mode 100644 index 000000000..75375b7a4 --- /dev/null +++ b/tests/lib/test_allele_detail.py @@ -0,0 +1,203 @@ +# ruff: noqa: E402 +"""Integration tests for the allele-detail assembly backing ``GET /alleles/{digest}`` (+ CAID). + +``get_allele_detail`` is the allele-grain sibling of ``get_variant_detail``: it anchors on an allele +(a digest, or a CAID's genomic+coding pair) and serves the **full cross-layer equivalence class** +(``get_allele_translations``), with each member labelled *relative to the focus* — faithful +(``projection``) vs. convergent cousin vs. reverse-translation ``candidate`` — plus the digest-keyed +annotations. These tests pin the labelling in both directions (nt focus / protein focus), the full-class +membership incl. cousins, the CAID focus set, the orphan fallback, and ``as_of``. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.models.allele import Allele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest) -> Allele: + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + assert allele is not None, f"allele {digest!r} not seeded" + return allele + + +def _coding_measured_specs(): + """A coding-measured record: authoritative cdna (+VEP+ClinGen), its genomic projection partner + (shared projection_group 0), the protein apex, and a synonymous cousin (own projection_group 1).""" + return [ + AlleleSpec( + digest="cdna-d", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-d", + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec(digest="prot-d", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr", post_mapped=_post_mapped()), + AlleleSpec( + digest="cousin-d", + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ] + + +@pytest.mark.integration +def test_nt_focus_labels_the_full_class_relative_to_the_queried_allele(session, setup_lib_db_with_score_set): + """Fetching the measured coding allele: the whole class comes back, each member labelled relative to + it — its genomic is a faithful coordinate projection, the protein its consequence, the cousin convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert detail.digest == "cdna-d" + assert detail.level == "cdna" + assert detail.vrs is not None and detail.vrs["type"] == "Allele" + assert set(detail.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} # full class, cousin included + + focus = detail.alleles["cdna-d"] + assert focus.is_focus is True and focus.relation is None and focus.derivation is None + assert focus.projection_of == "gen-d" + + genomic = detail.alleles["gen-d"] + assert genomic.is_focus is False + assert genomic.relation == "coordinate_representation_of" + assert genomic.derivation == "projection" + assert genomic.projection_of == "cdna-d" + + protein = detail.alleles["prot-d"] + assert protein.relation == "translation_of" + assert protein.derivation == "projection" + + cousin = detail.alleles["cousin-d"] + assert cousin.relation == "co_encodes" + assert cousin.derivation == "convergent" # distinct change sharing the consequence, not the coordinate partner + + +@pytest.mark.integration +def test_protein_focus_labels_every_nucleotide_as_a_candidate(session, setup_lib_db_with_score_set): + """Fetching the protein apex: walking down is ambiguous, so every nt member is a reverse-translation + candidate (the 'less power without a measurement' case, still labelled precisely by direction).""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "prot-d"), focus_digests={"prot-d"}) + + assert detail.alleles["prot-d"].is_focus is True + for nt in ("cdna-d", "gen-d", "cousin-d"): + assert detail.alleles[nt].relation == "encodes", nt + assert detail.alleles[nt].derivation == "candidate", nt + + +@pytest.mark.integration +def test_annotations_join_by_digest(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert set(detail.annotations) == set(detail.alleles) + assert detail.annotations["cdna-d"].vep is not None + assert detail.annotations["cdna-d"].vep.consequence == "missense_variant" + assert detail.annotations["gen-d"].vep is None # a member with no annotations still gets an empty block + + +@pytest.mark.integration +def test_caid_focus_flags_both_frames(session, setup_lib_db_with_score_set): + """A CAID names the nt change in both frames: fetched by CAID, the genomic + coding are both isFocus + (each other's coordinate partner), and only the true cousin is convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + # The CAID resolves to the {cdna, genomic} pair (both carry clingen_allele_id="CA1"). + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d", "gen-d"}) + + assert detail.alleles["cdna-d"].is_focus is True + assert detail.alleles["gen-d"].is_focus is True + assert detail.alleles["prot-d"].relation == "translation_of" + assert detail.alleles["cousin-d"].derivation == "convergent" + + +@pytest.mark.integration +def test_orphan_allele_falls_back_to_itself(session, setup_lib_db_with_score_set): + orphan = Allele(vrs_digest="orphan-d", level="cdna", hgvs_c="NM:c.9A>G", post_mapped=_post_mapped()) + session.add(orphan) + session.commit() + + detail = get_allele_detail(session, orphan, focus_digests={"orphan-d"}) + + assert set(detail.alleles) == {"orphan-d"} + assert detail.alleles["orphan-d"].is_focus is True + assert set(detail.annotations) == {"orphan-d"} + + +@pytest.mark.integration +def test_as_of_reconstructs_membership(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna", valid_from=T1) + + anchor = _allele(session, "cdna-d") + + before = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T0) + assert set(before.alleles) == {"cdna-d"} # links not yet live → class collapses to the focus + assert before.annotations["cdna-d"].vep is None + + after = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T2) + assert set(after.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} + assert after.annotations["cdna-d"].vep is not None diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py new file mode 100644 index 000000000..319caea27 --- /dev/null +++ b/tests/lib/test_allele_measurements.py @@ -0,0 +1,598 @@ +# ruff: noqa: E402 +"""Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). + +These pin the model: measurements aggregate the **cross-layer equivalence class** (co-membership in the +mapping/RT link graph), not one exact allele, and *which* allele anchors it determines the direct/related +split (a CA anchors on the nt alleles → nt measurements are direct, the protein consequence and the sibling +nt encodings are related; a PA anchors on the protein → protein measurements are direct, the nt encodings +related). Also: the protein apex reaches a consequence whose assay isn't reverse-translated (with its +cardinality/provenance instrumentation), the two authorization gates (score-set READ hides the measurement; +calibration READ withholds only the inline classification) exercised through the real ``has_permission``, +superseded opt-in, and the default ordering (direct-first, strongest evidence, pathogenic-biased). +""" + +import logging + +import pytest + +pytest.importorskip("psycopg2") + +from datetime import date + +from starlette_context import context, request_cycle_context + +from mavedb.lib.allele_measurements import ( + AlleleMeasurement, + MeasurementRelationship, + _ordering_key, + _resolve_protein_apex, + get_allele_measurements, +) +from mavedb.lib.types.authentication import UserData +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import EXTRA_USER, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER + + +def _user(session, username): + return session.query(User).filter(User.username == username).one() + + +def _user_data(session, username=TEST_USER["username"]): + """A ``UserData`` for a seeded user — the currency ``has_permission`` reads. ``None`` is anonymous.""" + return UserData(user=_user(session, username), active_roles=[]) + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + session.add( + MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + ) + session.commit() + + +def _calibration( + session, + score_set, + *, + variants, + functional="abnormal", + oddspaths_ratio=None, + primary=True, + private=False, + investigator_provided=False, + research_use_only=False, +): + user = _user(session, TEST_USER["username"]) + calibration = ScoreCalibration( + score_set_id=score_set.id, + title="cal", + primary=primary, + private=private, + investigator_provided=investigator_provided, + research_use_only=research_use_only, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label=functional, + functional_classification=functional, + oddspaths_ratio=oddspaths_ratio, + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +def _second_score_set( + session, base, urn, *, owner_username=TEST_USER["username"], private=True, superseded_score_set_id=None +): + """A sibling score set in the same experiment — for cross-score-set, private, and supersession tests. + ``superseded_score_set_id`` set on this (newer) row makes it the superseding version of that older + set (``older.superseding_score_set`` resolves back here).""" + owner = _user(session, owner_username) + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes", None) + score_set = ScoreSet( + **scaffold, + urn=urn, + experiment_id=base.experiment_id, + licence_id=base.licence_id, + superseded_score_set_id=superseded_score_set_id, + ) + score_set.private = private + score_set.created_by = owner + score_set.modified_by = owner + session.add(score_set) + session.commit() + session.refresh(score_set) + return score_set + + +def _seed_two_level_pair(session, score_set): + """One nt change (CA123) and its protein consequence (PA9), each measured in its own assay, with the + other level riding as a member — the minimal shape that exercises the direct/related asymmetry.""" + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna", hgvs_assay_level="NM_0:c.1A>T") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein", hgvs_assay_level="NP_0:p.Met1Leu") + _link(session, protein_record, prot, is_authoritative=True) + _link(session, protein_record, nt) + + return nt_measured, protein_measured + + +@pytest.mark.integration +def test_ca_entry_aggregates_direct_and_related(session, setup_lib_db_with_score_set): + """A CA page: the nt measurement is direct (assayed at this change); the protein measurement of its + consequence is related, labeled ``protein_consequence``, each carrying its own measured level.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert (by_urn[nt_measured.urn].relationship, by_urn[nt_measured.urn].assay_level) == ("direct", "cdna") + assert (by_urn[protein_measured.urn].relationship, by_urn[protein_measured.urn].assay_level) == ( + "protein_consequence", + "protein", + ) + # Assay HGVS pair rides through (mapped reference + submitted target). + assert by_urn[nt_measured.urn].assay_level_hgvs == "NM_0:c.1A>T" + assert by_urn[nt_measured.urn].submitted_hgvs == "c.1A>T" + assert by_urn[protein_measured.urn].submitted_hgvs == "p.Met1Leu" + + +@pytest.mark.integration +def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set): + """The same data anchored on the PA: the protein measurement is now direct and the nt measurement is + the related ``nucleotide_encoding`` — the asymmetry is purely which allele anchors the co-membership.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "direct" + assert by_urn[nt_measured.urn].relationship == "nucleotide_encoding" + + +@pytest.mark.integration +def test_sibling_nt_shown_on_ca_page(session, setup_lib_db_with_score_set): + """Reverse translation links every record to its full synonymous nt set, so a sibling nt change's record + links the anchor allele directly and the sibling shows on the CA page as ``nucleotide_encoding`` (after + the direct measurement). Both nt changes also show on the shared protein's PA page as encodings.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # Real RT linking: each record links its authoritative nt allele, the sibling nt allele (an RT + # candidate), and the shared protein consequence. + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, nt2) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, nt1) + _link(session, rc, prot) + + ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) + assert {m.variant_urn: m.relationship for m in ca_page} == {a.urn: "direct", c.urn: "nucleotide_encoding"} + # Direct precedes the sibling encoding. + assert [m.variant_urn for m in ca_page] == [a.urn, c.urn] + + pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} + assert all(m.relationship == "nucleotide_encoding" for m in pa_page) + + +@pytest.mark.integration +def test_resolve_protein_apex_single(session, setup_lib_db_with_score_set): + """The resolver returns the one protein consequence co-membered with the anchor nt allele, its PAID, + and no unregistered rows.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == ["PA9"] + assert apex.unregistered == 0 + + +@pytest.mark.integration +def test_resolve_protein_apex_unregistered(session, setup_lib_db_with_score_set): + """An apex protein allele with no CAID yet counts toward ``unregistered`` and contributes no PAID.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id=None) + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == [] + assert apex.unregistered == 1 + + +@pytest.mark.integration +def test_widening_ambiguous_apex_is_loud(session, setup_lib_db_with_score_set, caplog): + """An anchor whose nt change is co-membered with two *different* protein apexes (a divergent walk) is + surfaced loudly: a warning naming the conflicting PAIDs, and ``apex_paid_count == 2`` in the request + log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot1 = _allele(session, "prot-1", level="protein", clingen_allele_id="PA1") + prot2 = _allele(session, "prot-2", level="protein", clingen_allele_id="PA2") + + # nt is the measured allele of r1 (→ PA1) and a derived member of r2 (→ PA2): the same nt change + # resolving to two protein consequences. + v1 = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r1 = _record(session, v1, assay_level="cdna") + _link(session, r1, nt, is_authoritative=True) + _link(session, r1, prot1) + + v2 = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + r2 = _record(session, v2, assay_level="protein") + _link(session, r2, prot2, is_authoritative=True) + _link(session, r2, nt) + + with request_cycle_context({}): + with caplog.at_level(logging.WARNING, logger="mavedb.lib.allele_measurements"): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["apex_paid_count"] == 2 + + assert any("PA1" in rec.message and "PA2" in rec.message for rec in caplog.records) + + +@pytest.mark.integration +def test_widening_provenance_apex_only_unresolved(session, setup_lib_db_with_score_set): + """A protein measurement reached only through the apex, whose own record carries no derived link, is + counted apex-only-unresolved; giving that record a derived link of its own drops the count.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + # Both surface via the apex; the protein measurement is the apex-only-unresolved one. + assert {m.variant_urn for m in result} == {nt_measured.urn, protein_measured.urn} + assert context["measurements_protein_consequence"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + # Once the protein record gains a derived (non-authoritative) link of its own, it is resolved. + _link(session, protein_record, nt) + with request_cycle_context({}): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["measurements_apex_only_unresolved"] == 0 + + +@pytest.mark.integration +def test_ca_page_reaches_apex_only_protein_consequence(session, setup_lib_db_with_score_set): + """The variant page (no siblings) reaches a protein measurement of the anchor's consequence even when + that measurement's record has no nt link of its own (RT never ran on its score set) — through the + shared apex — and publishes the apex provenance to the request log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + by_urn = {m.variant_urn: m for m in result} + # Reached through the apex despite no nt link on the protein measurement's record. + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "protein_consequence" + # The page path now widens, so it emits the apex instrumentation. + assert context["apex_paid_count"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + +@pytest.mark.integration +def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): + """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, + even though it links the same (public, content-addressed) allele.""" + score_set = setup_lib_db_with_score_set # private, owned by TEST_USER + other = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", owner_username=EXTRA_USER["username"]) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + + visible = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, visible, assay_level="cdna"), nt, is_authoritative=True) + hidden = _variant(session, other, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, hidden, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert {m.variant_urn for m in result} == {visible.urn} + + +@pytest.mark.integration +def test_calibration_gate_withholds_classification(session, setup_lib_db_with_score_set): + """Calibration READ gates only the inline classification: on a public score set with a private + calibration, an anonymous caller still sees the measurement but its classification is withheld, while + the owner sees both.""" + score_set = setup_lib_db_with_score_set + public = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, public, variants=[variant], functional="abnormal", private=True) + + anonymous = get_allele_measurements(session, "CA123", user_data=None) + assert len(anonymous) == 1 + assert anonymous[0].preferred_classification is None + + owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert owner[0].preferred_classification is not None + assert owner[0].preferred_classification.functional_classification.value == "abnormal" + + +@pytest.mark.integration +def test_preferred_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): + """With no primary calibration, the preference cascade dominates evidence strength: an + investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) # plain, non-RUO + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0, investigator_provided=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. + assert result[0].preferred_classification.oddspaths_ratio == 2.0 + + +@pytest.mark.integration +def test_preferred_classification_strongest_within_tier(session, setup_lib_db_with_score_set): + """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_score_set): + """A research-use-only calibration is excluded from the cascade (even if it is the strongest + evidence).""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification is None + + +@pytest.mark.integration +def test_superseded_excluded_by_default_included_opt_in(session, setup_lib_db_with_score_set): + """Superseded measurements are current-tail-only by default; ``include_superseded`` opts in and they + self-describe (``is_current`` false + the superseding score set's URN).""" + score_set = setup_lib_db_with_score_set + _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", superseded_score_set_id=score_set.id) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + assert get_allele_measurements(session, "CA123", user_data=_user_data(session)) == [] + + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_superseded=True) + assert len(opted) == 1 + assert opted[0].is_current is False + assert opted[0].superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +@pytest.mark.integration +def test_unreadable_superseding_reads_as_current(session, setup_lib_db_with_score_set): + """A superseding version the caller cannot read is not leaked — the measurement reads as current and + is included by default.""" + score_set = setup_lib_db_with_score_set # owned by TEST_USER + _second_score_set( + session, + score_set, + "urn:mavedb:00000001-a-2", + owner_username=EXTRA_USER["username"], + superseded_score_set_id=score_set.id, + ) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert len(result) == 1 + assert result[0].is_current is True + assert result[0].superseded_by_score_set is None + + +@pytest.mark.integration +def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_with_score_set): + """Default order: direct before related; within a group strongest evidence first, pathogenic before + benign at equal magnitude; the unclassified related measurement sorts last.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + strong_path = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_path], functional="abnormal", oddspaths_ratio=100.0) + + strong_benign = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_benign, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_benign], functional="normal", oddspaths_ratio=0.01) + + weak_path = _variant(session, score_set, 3, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, weak_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[weak_path], functional="abnormal", oddspaths_ratio=2.0) + + related = _variant(session, score_set, 4, data={"score_data": {"score": 1.0}}) + related_record = _record(session, related, assay_level="protein") + _link(session, related_record, prot, is_authoritative=True) + _link(session, related_record, nt) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert [m.variant_urn for m in result] == [strong_path.urn, strong_benign.urn, weak_path.urn, related.urn] + + +@pytest.mark.integration +def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): + assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] + + +def _oddspaths_classification(ratio): + """A transient functional classification carrying only an oddspaths ratio — enough for + ``classification_evidence_strength`` (which reads ``acmg_classification`` [None here] then the ratio) + to rank it, with no DB round-trip.""" + return ScoreCalibrationFunctionalClassification(oddspaths_ratio=ratio) + + +def _measurement(urn, *, relationship=MeasurementRelationship.direct, classification=None, is_current=True): + """A transit ``AlleleMeasurement`` with just the fields ``_ordering_key`` reads; the rest are filler.""" + return AlleleMeasurement( + variant_urn=urn, + score=None, + assay_level=None, + relationship=relationship, + assay_level_hgvs=None, + submitted_hgvs=None, + score_set_urn="", + score_set_title="", + preferred_classification=classification, + is_current=is_current, + superseded_by_score_set=None, + ) + + +def test_ordering_key_ranks_every_dimension(): + """Pin the full precedence chain of ``_ordering_key`` as a pure unit (no DB, no calibration fixtures): + current → direct → classified → strongest magnitude → pathogenic direction → newest published → urn. + Each adjacent pair below differs by exactly the next-lower key, so sorting a shuffled copy must + reproduce this exact order.""" + strong_path = _oddspaths_classification(100.0) # magnitude |ln 100|, pathogenic + strong_benign = _oddspaths_classification(0.01) # equal magnitude, benign + weak_path = _oddspaths_classification(2.0) # smaller magnitude, pathogenic + + # (measurement, published_date), already in the expected ranked order. + ranked = [ + (_measurement("u#1", classification=strong_path), date(2024, 1, 1)), # strongest evidence, pathogenic + (_measurement("u#2", classification=strong_benign), date(2024, 1, 1)), # equal magnitude, benign loses + (_measurement("u#3", classification=weak_path), date(2024, 1, 1)), # weaker magnitude + (_measurement("u#4a", classification=None), date(2024, 6, 1)), # unclassified; newer published leads its tie + (_measurement("u#4b", classification=None), date(2024, 1, 1)), # unclassified, older published + (_measurement("u#4c", classification=None), date(2024, 1, 1)), # unclassified, same date → urn tiebreak + (_measurement("u#5", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1)), # related + (_measurement("u#6", is_current=False), date(2024, 6, 1)), # superseded sorts last + ] + + shuffled = [ranked[i] for i in (5, 0, 7, 2, 4, 1, 6, 3)] + ordered = sorted(shuffled, key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == [m.variant_urn for m, _ in ranked] + + +def test_ordering_key_missing_published_date_sorts_after_dated(): + """A null published date is treated as oldest (``0`` ordinal), so a dated measurement leads an + otherwise-identical undated one.""" + dated = (_measurement("u#dated"), date(2024, 1, 1)) + undated = (_measurement("u#undated"), None) + + ordered = sorted([undated, dated], key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == ["u#dated", "u#undated"] diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py new file mode 100644 index 000000000..3e3a15055 --- /dev/null +++ b/tests/lib/test_alleles.py @@ -0,0 +1,164 @@ +# ruff: noqa: E402 +"""Integration tests for the record-scoped allele-link query backing Cat-VRS transit. + +``get_live_record_allele_links`` stays within one variant's own live ``MappingRecord`` — unlike +``get_allele_translations``, which takes the cross-record union an anchor allele can belong to. These +tests pin that scoping, the ValidTime live/retired filtering, and the ``as_of`` reconstruction. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock, so an accidental func.now() stamp is visibly +# wrong rather than coincidentally equal (mirrors tests/db/test_mixins.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _record(session, variant, *, assay_level="genomic", valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +def _digests(links): + return {link.allele.vrs_digest for link in links} + + +@pytest.mark.integration +def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): + """The variant's live record yields its authoritative + derived links, each allele eagerly loaded.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + authoritative = _allele(session, "auth", level="genomic") + derived = _allele(session, "deriv", level="protein") + _link(session, record, authoritative, is_authoritative=True) + _link(session, record, derived, is_authoritative=False) + + links = get_live_record_allele_links(session, variant.id) + + assert _digests(links) == {"auth", "deriv"} + # Exactly one authoritative (the defining allele); the eager-loaded allele is reachable. + assert [link.allele.vrs_digest for link in links if link.is_authoritative] == ["auth"] + + +@pytest.mark.integration +def test_no_mapping_record_returns_empty(session, setup_lib_db_with_score_set): + assert get_live_record_allele_links(session, 1) == [] + + +@pytest.mark.integration +def test_excludes_a_retired_link(session, setup_lib_db_with_score_set): + """A link retired within a still-live record is not part of the current set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + record = _record(session, variant) + live = _allele(session, "live") + stale = _allele(session, "stale") + _link(session, record, live, is_authoritative=True) + retired = _link(session, record, stale) + retired.retire(at=T1) + session.commit() + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"live"} + + +@pytest.mark.integration +def test_excludes_a_superseded_record(session, setup_lib_db_with_score_set): + """A re-map retires the prior record (cascading to its links); only the new record's links surface.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + old_allele = _allele(session, "old") + new_allele = _allele(session, "new") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"new"} + + +@pytest.mark.integration +def test_record_scoped_not_cross_record_union(session, setup_lib_db_with_score_set): + """A shared allele linked by two variants' records does NOT pull the other record's members in — + this is the deliberate difference from get_allele_translations' cross-record union.""" + score_set = setup_lib_db_with_score_set + shared = _allele(session, "shared") + only_a = _allele(session, "only_a") + only_b = _allele(session, "only_b") + + variant_a = _variant(session, score_set, 1) + record_a = _record(session, variant_a) + _link(session, record_a, shared, is_authoritative=True) + _link(session, record_a, only_a) + + variant_b = _variant(session, score_set, 2) + record_b = _record(session, variant_b) + _link(session, record_b, shared, is_authoritative=True) + _link(session, record_b, only_b) + + # Variant A's record sees the shared allele and its own member, never variant B's. + assert _digests(get_live_record_allele_links(session, variant_a.id)) == {"shared", "only_a"} + + +@pytest.mark.integration +def test_as_of_returns_the_historical_link_set(session, setup_lib_db_with_score_set): + """as_of reconstructs the record + links live at a past instant, not the current re-mapped set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + past_allele = _allele(session, "past") + current_allele = _allele(session, "current") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, past_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, current_allele, is_authoritative=True, valid_from=T1) + + # Inside the old window: the historical set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T0)) == {"past"} + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} + # At/after the handoff and current: the new set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T2)) == {"current"} + assert _digests(get_live_record_allele_links(session, variant.id)) == {"current"} diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index 52771b6bf..354ed85fc 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -5,1362 +5,278 @@ pytest.importorskip("psycopg2") from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.models.allele import Allele from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus -from mavedb.models.variant import Variant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason @pytest.fixture def annotation_status_manager(session, job_run): - """Fixture to provide an AnnotationStatusManager instance.""" return AnnotationStatusManager(session, job_run_id=job_run.id) @pytest.fixture -def existing_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.fixture -def existing_unversioned_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() +def allele(session): + allele = Allele(vrs_digest="asm-test-allele-digest", level="genomic") + session.add(allele) session.commit() + session.refresh(allele) + return allele - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.mark.unit -class TestAnnotationStatusManagerCreateAnnotationUnit: - """Unit tests for AnnotationStatusManager.add_annotation method.""" - - @pytest.mark.parametrize( - "annotation_type", - AnnotationType._member_map_.values(), - ) - @pytest.mark.parametrize( - "status", - AnnotationStatus._member_map_.values(), - ) - def test_add_annotation_creates_entry_with_annotation_type_version_status( - self, session, annotation_status_manager, annotation_type, status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry with correct type and version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - annotation_data={}, - current=True, - status=status, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - ) - - assert annotation is not None - assert annotation.annotation_type == annotation_type - assert annotation.status == status - assert annotation.version == "v1.0" - - def test_add_annotation_stores_job_run_id( - self, session, annotation_status_manager, job_run, setup_lib_db_with_variant - ): - """Test that every annotation is created with the job_run_id from the manager.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - annotation_data={}, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.job_run_id == job_run.id - - def test_add_annotation_persists_annotation_data( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding an annotation persists the provided annotation data.""" - annotation_data = { - "annotation_metadata": {"some_key": "some_value"}, - "error_message": None, - } - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - failure_category=None, - annotation_data=annotation_data, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.failure_category is None - for key, value in annotation_data.items(): - assert getattr(annotation, key) == value - - def test_add_annotation_creates_entry_and_marks_previous_not_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry and marks previous ones as not current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type, version) - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - def test_add_annotation_with_different_version_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different version keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type) but different version - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_with_different_type_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different type keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same variant but different type - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_without_version(self, session, annotation_status_manager, setup_lib_db_with_variant): - """Test that adding an annotation without specifying version works correctly.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - version=None, - annotation_data={}, - status=AnnotationStatus.SKIPPED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.version is None - assert annotation.current is True - - def test_add_annotation_multiple_without_version_marks_previous_not_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding multiple annotations without version marks previous ones as not current.""" - - # Add second annotation without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is False - - def test_add_annotation_different_type_without_version_keeps_previous_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation of different type without version keeps previous current.""" - - # Add second annotation of different type without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is True - - def test_add_annotation_multiple_variants_independent_current_flags( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants maintains independent current flags.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation1 is not None - assert annotation1.id is not None - assert annotation1.current is True - - assert annotation2 is not None - assert annotation2.id is not None - assert annotation2.current is True - - -class TestAnnotationStatusManagerGetCurrentAnnotationUnit: - """Unit tests for AnnotationStatusManager.get_current_annotation method.""" - - def test_get_current_annotation_returns_none_when_no_entry( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when no entry exists.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_returns_correct_entry( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation.id == existing_annotation_status.id - assert annotation.current is True - - def test_get_current_annotation_returns_none_for_non_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when the entry is not current.""" - # Mark existing annotation as not current - existing_annotation_status.current = False - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_with_different_version_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different version returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert annotation is None - - def test_get_current_annotation_with_different_type_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different type returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_without_version_returns_correct_entry( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation without version returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - ) - assert annotation.id == existing_unversioned_annotation_status.id - assert annotation.current is True - - -class TestAnnotationStatusManagerIntegration: - """Integration tests for AnnotationStatusManager methods.""" - - def test_add_and_get_current_annotation_work_together( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding and getting current annotation work together correctly.""" - # Add annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_multiple_and_get_current_returns_latest( - self, session, annotation_status_manager, version, setup_lib_db_with_variant - ): - """Test that adding multiple annotations and getting current returns the latest one.""" - # Add first annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Add second annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.version == version - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_annotations_for_different_variants_and_get_current_independent( - self, session, annotation_status_manager, version, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants and getting current works independently.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation for variant 1 - retrieved_annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation1 is not None - assert retrieved_annotation1.current is True - assert retrieved_annotation1.status == AnnotationStatus.SUCCESS - assert retrieved_annotation1.version == version - - # Get current annotation for variant 2 - retrieved_annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation2 is not None - assert retrieved_annotation2.current is True - assert retrieved_annotation2.status == AnnotationStatus.FAILED - assert retrieved_annotation2.version == version - - -@pytest.mark.unit -class TestAnnotationStatusManagerReplaceAllVersionsUnit: - """Unit tests for the replace_all_versions parameter of AnnotationStatusManager.add_annotation.""" - - def test_replace_all_versions_false_keeps_different_version_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Default behavior: a new annotation only retires the same version, not others.""" - # existing_annotation_status is version "v1", current=True - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_annotation is not None - assert new_annotation.current is True - - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_replace_all_versions_true_retires_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True retires all current records for (variant, type) regardless of version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - # Both v1 and v2 are current at this point (replace_all_versions=False) - v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert v1 is not None and v1.current is True - assert v2 is not None and v2.current is True - - # Now add v3 with replace_all_versions=True — should retire both v1 and v2 - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(v1) - session.refresh(v2) - assert v1.current is False - assert v2.current is False - - v3 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - ) - assert v3 is not None and v3.current is True - def test_replace_all_versions_true_only_affects_matching_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True only retires records for the same annotation_type.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( +class TestRecordEvent: + def test_records_variant_subject_event(self, session, annotation_status_manager, setup_lib_db_with_variant): + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) annotation_status_manager.flush() session.commit() - vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", + event = annotation_status_manager.get_current_annotation( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id ) + assert event is not None + assert event.disposition == Disposition.PRESENT + assert event.allele_id is None + # No current flag exists on the event log. + assert not hasattr(event, "current") - # replace VRS_MAPPING only - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + def test_records_allele_subject_event_with_metadata(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + metadata={"gnomad_variant_id": "1-55051215-G-A"}, ) annotation_status_manager.flush() session.commit() - session.refresh(vrs) - session.refresh(clinvar) - assert vrs.current is False - assert clinvar.current is True - - new_vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_vrs is not None and new_vrs.current is True - - def test_replace_all_versions_true_only_affects_matching_variant( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """replace_all_versions=True only retires records for the same variant_id.""" - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - ann2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - # replace variant1 only - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() - session.commit() + assert event.event_metadata == {"gnomad_variant_id": "1-55051215-G-A"} + assert event.source_version == "4.1.0" + assert event.variant_id is None - session.refresh(ann1) - session.refresh(ann2) - assert ann1.current is False - assert ann2.current is True # untouched - new_ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_ann1 is not None and new_ann1.current is True - - def test_replace_all_versions_true_same_version_also_retired( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """replace_all_versions=True retires a same-version record just as replace_all_versions=False would.""" - # existing_annotation_status is version "v1" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=True, - ) +class TestAppendOnlyLatestWins: + def test_latest_event_by_id_is_current(self, session, annotation_status_manager, allele): + for reason in (EventReason.CREATED, EventReason.RECONFIRMED, EventReason.SKIPPED): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=reason, + source_version="4.1.0", + ) annotation_status_manager.flush() session.commit() - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + # All three events persist (append-only — no retire of prior rows). + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert new_annotation is not None - assert new_annotation.current is True - assert new_annotation.status == AnnotationStatus.FAILED - - -@pytest.mark.unit -class TestAnnotationStatusManagerBatchingUnit: - """Unit tests for batching and flush behavior.""" - - def test_flush_noop_when_empty(self, annotation_status_manager): - """flush() with no pending annotations does nothing and does not error.""" - annotation_status_manager.flush() # should not raise - - def test_auto_flush_at_batch_size(self, session, setup_lib_db_with_score_set): - """Annotations are auto-flushed to the DB when batch_size is reached.""" - variants = [ - Variant(score_set_id=1, hgvs_nt=f"NM_000000.1:c.{i}A>G", hgvs_pro=f"NP_000000.1:p.Met{i}Val", data={}) - for i in range(3) + assert [e.reason for e in history] == [ # newest first + EventReason.SKIPPED, + EventReason.RECONFIRMED, + EventReason.CREATED, ] - session.add_all(variants) - session.commit() - for v in variants: - session.refresh(v) - - manager = AnnotationStatusManager(session, batch_size=2) - - # Add first — stays pending (below threshold) - manager.add_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Add second — triggers auto-flush (reaches batch_size=2) - manager.add_annotation( - variant_id=variants[1].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 0 # flushed - - # Verify the auto-flushed rows are visible in the DB - ann = manager.get_current_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann is not None and ann.current is True - - # Add a third — stays pending (below threshold again) - manager.add_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - # Explicit flush persists the remainder - manager.flush() - assert len(manager._pending) == 0 - - ann3 = manager.get_current_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann3 is not None and ann3.current is True - - def test_get_current_annotation_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_current_annotation() flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush — get_current_annotation should auto-flush - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is not None - assert annotation.current is True + # Current = newest by id. + current = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert current.reason == EventReason.SKIPPED + + def test_source_version_scopes_current(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.0.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + v40 = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id, source_version="4.0.0" + ) + assert v40.disposition == Disposition.PRESENT + + +class TestSubjectValidation: + def test_variant_subject_type_with_allele_id_raises(self, annotation_status_manager, allele): + with pytest.raises(ValueError, match="variant-subject"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_allele_subject_type_with_variant_id_raises(self, annotation_status_manager, setup_lib_db_with_variant): + with pytest.raises(ValueError, match="allele-subject"): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_neither_subject_raises(self, annotation_status_manager): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with_variant, allele): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + +class TestQueryReads: + def test_get_current_annotation_returns_none_when_no_event(self, annotation_status_manager, allele): + # No event recorded for this subject yet — current status is None, not an error. + assert ( + annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + is None + ) + + def test_get_event_history_empty_for_no_records(self, annotation_status_manager, allele): + assert ( + annotation_status_manager.get_event_history(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id) + == [] + ) + + def test_flush_is_noop_when_empty(self, annotation_status_manager): + # Nothing pending — flush must not raise or emit a write. assert len(annotation_status_manager._pending) == 0 - - def test_flush_clears_internal_buffers(self, session, annotation_status_manager, setup_lib_db_with_variant): - """flush() clears both _pending and _retirement_filters.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(annotation_status_manager._pending) == 1 - assert len(annotation_status_manager._retirement_filters) == 1 - annotation_status_manager.flush() assert len(annotation_status_manager._pending) == 0 - assert len(annotation_status_manager._retirement_filters) == 0 - - def test_batch_retirement_groups_by_annotation_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Multiple annotation types in one batch are retired independently.""" - # Create initial annotations for two types - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - vrs_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # Now add replacements for both types in one batch - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs_v1) - session.refresh(clinvar_v1) - assert vrs_v1.current is False - assert clinvar_v1.current is False - - vrs_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - clinvar_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - ) - assert vrs_v2 is not None and vrs_v2.current is True - assert clinvar_v2 is not None and clinvar_v2.current is True - - -@pytest.mark.unit -class TestAnnotationStatusManagerAuditHelpersUnit: - """Unit tests for audit query helpers: get_annotation_history and get_all_current_annotations.""" - - def test_get_annotation_history_returns_all_rows_newest_first( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history returns both current and retired rows, newest first.""" - # Create two annotations for the same (variant, type, version) — first gets retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert len(history) == 2 - # Newest first - assert history[0].status == AnnotationStatus.FAILED - assert history[0].current is True - assert history[1].status == AnnotationStatus.SUCCESS - assert history[1].current is False - - def test_get_annotation_history_filters_by_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history with version only returns matching rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history_jan = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - ) - assert len(history_jan) == 1 - assert history_jan[0].version == "2025-01" - - def test_get_annotation_history_without_version_returns_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history without version returns rows across all versions.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, + def test_get_current_annotation_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, ) - assert len(history) == 2 - - def test_get_annotation_history_empty_for_no_records(self, annotation_status_manager, setup_lib_db_with_variant): - """get_annotation_history returns empty list when no records exist.""" - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, + # Buffered, never explicitly flushed — the read must flush first and still see it. + assert len(annotation_status_manager._pending) == 1 + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert history == [] + assert event is not None + assert len(annotation_status_manager._pending) == 0 - def test_get_annotation_history_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_get_event_history_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, ) - # No explicit flush - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, + assert len(annotation_status_manager._pending) == 1 + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) assert len(history) == 1 assert len(annotation_status_manager._pending) == 0 - def test_get_all_current_annotations_returns_all_types( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns current annotations across all types.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_failed_disposition_round_trips_through_manager(self, session, annotation_status_manager, allele): + # The failure path (the case the audit log most needs to capture) survives a write/read cycle + # with its disposition, reason, and error metadata intact. + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "upstream timeout"}, ) annotation_status_manager.flush() session.commit() - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - types = {a.annotation_type for a in all_current} - assert types == { - AnnotationType.VRS_MAPPING, - AnnotationType.CLINVAR_CONTROL, - AnnotationType.CLINGEN_ALLELE_ID, - } - - def test_get_all_current_annotations_excludes_retired( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations does not include retired rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.event_metadata == {"error_message": "upstream timeout"} - # Replace it — v1 becomes retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() + def test_current_is_isolated_per_subject(self, session, annotation_status_manager, allele): + # Recording for one allele must not bleed into another's current status. + other = Allele(vrs_digest="asm-test-allele-digest-2", level="genomic") + session.add(other) session.commit() + session.refresh(other) - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 1 - assert all_current[0].version == "v2" - - def test_get_all_current_annotations_empty_for_no_records( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns empty list when no records exist.""" - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert result == [] - - def test_get_all_current_annotations_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(result) == 1 - assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_ordered_by_type_then_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns results ordered by annotation_type, version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=other.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", ) annotation_status_manager.flush() session.commit() - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, + a = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert len(all_current) == 3 - # clinvar_control < vrs_mapping alphabetically - assert all_current[0].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[0].version == "2025-01" - assert all_current[1].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[1].version == "2025-02" - assert all_current[2].annotation_type == AnnotationType.VRS_MAPPING + b = annotation_status_manager.get_current_annotation(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=other.id) + assert a.disposition == Disposition.PRESENT + assert b.disposition == Disposition.ABSENT + assert a.allele_id != b.allele_id -@pytest.mark.unit -class TestVariantAnnotationStatusReprUnit: - """Unit tests for the VariantAnnotationStatus __repr__ method.""" - - def test_repr_includes_key_fields(self, session, annotation_status_manager, setup_lib_db_with_variant): - """__repr__ includes id, variant_id, type, version, status, current, and created_at.""" - annotation_status_manager.add_annotation( +class TestBatching: + def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): + annotation_status_manager.batch_size = 2 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( + # Not yet flushed (1 < batch_size). + assert len(annotation_status_manager._pending) == 1 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - repr_str = repr(annotation) - - assert "VariantAnnotationStatus" in repr_str - assert f"id={annotation.id}" in repr_str - assert f"variant_id={setup_lib_db_with_variant.id}" in repr_str - assert "type='vrs_mapping'" in repr_str - assert "version='v1'" in repr_str - assert "status='success'" in repr_str - assert "current=True" in repr_str - assert "created_at=" in repr_str + # Auto-flushed at batch_size=2. + assert len(annotation_status_manager._pending) == 0 + session.commit() diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py new file mode 100644 index 000000000..1ee2b9ba0 --- /dev/null +++ b/tests/lib/test_cat_vrs.py @@ -0,0 +1,421 @@ +# ruff: noqa: E402 +"""Tests for the on-the-fly Cat-VRS transit builder. + +The pure builder is unit-tested over transient ``MappingRecordAllele`` instances (no DB) — asserting +the mode, the member->defining relations, and the spec-pure ``CategoricalVariant`` shape for both +score-collapse modes. The DB-backed wrapper ``categorical_variant_for_variant`` is exercised at the +bottom against a real session to pin the fetch + ``as_of`` threading. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, Relation +from ga4gh.core.models import Relation as MappingRelation + +from mavedb.lib.cat_vrs import ( + _SPEC_EQUIVALENT, + _relation_concept, + CatVrsMode, + CatVrsRelation, + build_categorical_variant, + categorical_variant_for_variant, +) +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# A spec-valid 32-char VRS digest; the internal VRS digest is irrelevant to the builder, which keys +# member relations on the `vrs_digest` *column*, so one fixed valid value across members is fine. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A minimal but spec-valid post_mapped VRS Allele dict.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +_DEFAULT_POST_MAPPED = object() + + +def _link( + *, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED, projection_group=None +) -> MappingRecordAllele: + """A transient (record, allele) link with its allele attached — no session needed. + + ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), + deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass + ``post_mapped=None`` to model an un-hydratable allele. ``projection_group`` pairs a c↔g projection + (the two links of one precise change share a value; the protein apex carries ``None``); the builder + uses it in projection mode to keep only the measured change's precise coordinate partner. + """ + pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped + allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) + + +@pytest.mark.unit +def test_mode_2_protein_measured_reverse_translation(): + """Protein measured: defining is the protein allele; both nt members `encodes` it (star model).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # Per-member relations exclude the defining allele; both nt siblings encode the protein. + assert transit.member_relations == { + "cdna": CatVrsRelation.ENCODES, + "gen": CatVrsRelation.ENCODES, + } + + cv = transit.categorical_variant + assert isinstance(cv, CategoricalVariant) + # Every representation is a member, including the defining protein allele. + assert len(cv.members) == 3 + # Constraints come back as the `Constraint` union RootModel; unwrap to the concrete type. + constraint = cv.constraints[0].root + assert isinstance(constraint, DefiningAlleleConstraint) + # Relations on the constraint carry only the distinct kinds present (here: just `encodes`). + assert [str(r.primaryCoding.code.root) for r in constraint.relations] == ["encodes"] + + +@pytest.mark.unit +def test_mode_1_coding_measured_projection(): + """Coding measured: the projection_group partner is a coordinate representation; protein is a translation.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2") + assert transit is not None + + assert transit.mode == CatVrsMode.PROJECTION + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of"} + + +@pytest.mark.unit +def test_mode_1_projection_includes_sibling_encoders(): + """Coding measured, full closure (default include_convergent=True): the reverse-translation fan on the + record (other encoders of the same protein consequence, in *different* projection groups) is kept as + members wearing the ``co_encodes`` relation, so the object is the full closure. The measured change's + own coordinate partner and protein consequence stay coordinate_representation_of / translation_of.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + # A sibling encoder of the same protein change — a distinct variant, different projection group. + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b") + assert transit is not None + + # The coordinate partner + protein consequence keep their faithful relations; the two cousins ride as + # co_encodes (distinct, unmeasured synonymous variants). + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + "sibling_cdna": CatVrsRelation.CO_ENCODES, + "sibling_gen": CatVrsRelation.CO_ENCODES, + } + # members = defining cdna + gen partner + protein apex + the two cousins (full closure). + assert len(transit.categorical_variant.members) == 5 + # The distinct relation kinds surface on the constraint, including co_encodes. + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of", "co_encodes"} + + +@pytest.mark.unit +def test_mode_1_projection_narrow_object_drops_sibling_encoders(): + """Coding measured, narrow object (include_convergent=False, the VA subject): the synonymous cousins + in other projection groups are dropped — only the measured change's coordinate partner and protein + consequence remain. This pins the shape the VA-Spec path builds.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b-narrow", include_convergent=False) + assert transit is not None + + # Only the measured change's coordinate partner + the protein consequence; the cousins are excluded. + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + # members = defining cdna + gen partner + protein apex (the two cousins are dropped). + assert len(transit.categorical_variant.members) == 3 + + +@pytest.mark.unit +def test_mode_2_reverse_translation_keeps_the_full_encoder_class(): + """Protein measured: every nt encoder stays, across projection groups — the full equivalence class + is what the protein claim ranges over (no sibling filtering in reverse-translation mode).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0), + _link(level="genomic", digest="gen_a", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="gen_b", is_authoritative=False, projection_group=1), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2c") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # All four nt encoders `encodes` the defining protein; none dropped. + assert transit.member_relations == { + "cdna_a": CatVrsRelation.ENCODES, + "gen_a": CatVrsRelation.ENCODES, + "cdna_b": CatVrsRelation.ENCODES, + "gen_b": CatVrsRelation.ENCODES, + } + assert len(transit.categorical_variant.members) == 5 + + +@pytest.mark.unit +def test_no_authoritative_link_returns_none(): + """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" + links = [_link(level="genomic", digest="gen", is_authoritative=False)] + assert build_categorical_variant(links, name="urn:mavedb:test#3") is None + + +@pytest.mark.unit +def test_empty_links_returns_none(): + assert build_categorical_variant([], name="urn:mavedb:test#4") is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + "defining_level, expected_mode", + [ + ("protein", CatVrsMode.REVERSE_TRANSLATION), + ("cdna", CatVrsMode.PROJECTION), + ("genomic", CatVrsMode.PROJECTION), + ], +) +def test_mode_follows_defining_level(defining_level, expected_mode): + links = [_link(level=defining_level, digest="d", is_authoritative=True)] + transit = build_categorical_variant(links, name="urn:mavedb:test#5") + assert transit is not None + assert transit.mode == expected_mode + + +@pytest.mark.unit +def test_unhydratable_defining_allele_returns_none(): + """Defensive: an authoritative allele with no post_mapped can't anchor a Cat-VRS, so build → None + (a broken invariant in practice — the mapping job writes post_mapped on the authoritative allele).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, post_mapped=None), + _link(level="cdna", digest="cdna", is_authoritative=False), + ] + assert build_categorical_variant(links, name="urn:mavedb:test#6") is None + + +@pytest.mark.unit +def test_unhydratable_member_allele_is_skipped(): + """Defensive: a member with no post_mapped is dropped; the build still succeeds on the rest.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="good", is_authoritative=False), + _link(level="genomic", digest="bad", is_authoritative=False, post_mapped=None), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#7") + + assert transit is not None + # The un-hydratable genomic member is excluded from both the members and the relation map. + assert transit.member_relations == {"good": CatVrsRelation.ENCODES} + assert len(transit.categorical_variant.members) == 2 + + +# --- DB-backed wrapper: categorical_variant_for_variant (fetch + build, as_of threaded) --- + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _db_allele(session, digest, level): + allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped()) + session.add(allele) + session.commit() + return allele + + +def _db_variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _db_record(session, variant, *, assay_level, valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _db_link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +@pytest.mark.integration +def test_wrapper_builds_transit_from_live_links(session, setup_lib_db_with_score_set): + """Fetch + build: a protein-measured variant's record yields a Mode 2 transit with nt `encodes`.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + record = _db_record(session, variant, assay_level="protein") + _db_link(session, record, _db_allele(session, "prot", "protein"), is_authoritative=True) + _db_link(session, record, _db_allele(session, "cdna", "cdna")) + + transit = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert transit is not None + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + assert transit.member_relations == {"cdna": CatVrsRelation.ENCODES} + + +@pytest.mark.integration +def test_wrapper_returns_none_for_unmapped_variant(session, setup_lib_db_with_score_set): + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + assert categorical_variant_for_variant(session, variant.id, name=variant.urn) is None + + +@pytest.mark.integration +def test_wrapper_threads_as_of_to_the_historical_record(session, setup_lib_db_with_score_set): + """as_of selects the past record, changing the built object — Mode 2 then, Mode 1 now.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + + old_record = _db_record(session, variant, assay_level="protein", valid_from=T0) + _db_link(session, old_record, _db_allele(session, "old-prot", "protein"), is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _db_record(session, variant, assay_level="genomic", valid_from=T1) + _db_link(session, new_record, _db_allele(session, "new-gen", "genomic"), is_authoritative=True, valid_from=T1) + + past = categorical_variant_for_variant(session, variant.id, name=variant.urn, as_of=T0) + current = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert past is not None and past.mode == CatVrsMode.REVERSE_TRANSLATION + assert current is not None and current.mode == CatVrsMode.PROJECTION + + +# --------------------------------------------------------------------------- +# Spec alignment +# +# MaveDB maintains its own relation vocabulary (Cat-VRS's `Relation` is an enum with members, so it +# cannot be subclassed, and inheriting would buy nothing — interop rides on `Coding.system`, not on +# Python enum identity). These tests are the drift guard that keeps the vocabulary honest: they fail +# when the spec gains a term MaveDB has not triaged, or when MaveDB coins a term without recording +# whether the spec already covers it. +# --------------------------------------------------------------------------- + + +def test_mapped_relations_emit_an_exact_match_to_the_spec_term(): + """A code with a spec equivalent must carry a machine-readable mapping to it. + + A matching code *string* is not enough: the two codings sit in different systems, so nothing but an + explicit ConceptMapping tells a consumer they are the same concept. + """ + concept = _relation_concept(CatVrsRelation.TRANSLATION_OF) + + assert concept.primaryCoding is not None + assert concept.primaryCoding.system == "https://mavedb.org/cat-vrs/relations" + assert concept.mappings is not None and len(concept.mappings) == 1 + + mapping = concept.mappings[0] + assert mapping.relation == MappingRelation.EXACT_MATCH + assert mapping.coding.code.root == Relation.TRANSLATION_OF.value + assert mapping.coding.system != concept.primaryCoding.system + + +@pytest.mark.parametrize( + "relation", + [CatVrsRelation.ENCODES, CatVrsRelation.CO_ENCODES, CatVrsRelation.COORDINATE_REPRESENTATION_OF], +) +def test_unmapped_relations_carry_no_mapping(relation): + """The absence of a mapping is the interoperable statement that no spec term covers this code. + + Emitting a `closeMatch` to something approximate would be worse than silence — a consumer would + resolve it and be wrong. See `_SPEC_EQUIVALENT` for why each of these three has no equivalent. + """ + concept = _relation_concept(relation) + + assert concept.primaryCoding is not None and concept.primaryCoding.code.root == relation.value + assert concept.mappings is None + + +def test_every_relation_is_either_mapped_or_a_declared_gap(): + """No MaveDB relation may exist without a triage decision recorded in `_SPEC_EQUIVALENT`. + + Coining a new code is fine; coining one *without deciding whether the spec already covers it* is the + drift this guards against. A new member fails here until it is either mapped or listed below. + """ + declared_gaps = { + CatVrsRelation.ENCODES, + CatVrsRelation.CO_ENCODES, + CatVrsRelation.COORDINATE_REPRESENTATION_OF, + } + + assert set(CatVrsRelation) == set(_SPEC_EQUIVALENT) | declared_gaps + + +def test_spec_relations_maveDB_deliberately_never_emits(): + """Fails when Cat-VRS publishes a relation MaveDB has not triaged. + + `liftover_to` is assembly-to-assembly, which MaveDB does not do. `transcribed_to` is genomic-> + transcript only and asserts transcription; MaveDB's c<->g relation is direction-neutral coordinate + equivalence that also holds for intronic/UTR-offset positions, so it is deliberately unmapped. + """ + emitted = {spec_term.value for spec_term in _SPEC_EQUIVALENT.values()} + never_emitted = {"liftover_to", "transcribed_to"} + + assert {relation.value for relation in Relation} == emitted | never_emitted diff --git a/tests/lib/test_clinical_controls.py b/tests/lib/test_clinical_controls.py new file mode 100644 index 000000000..194ff68bd --- /dev/null +++ b/tests/lib/test_clinical_controls.py @@ -0,0 +1,32 @@ +"""Unit tests for the clinical-controls lib helpers (``lib/clinical_controls.py``). + +The live-link query paths are exercised end-to-end by the score-set router tests; these pin the pure +``clinvar_version_sort_key``, which is the ``MM_YYYY`` release-ordering contract the UI parses in +parallel (``lib/clinical-controls.ts``) and so must stay stable. +""" + +from mavedb.lib.clinical_controls import clinvar_version_sort_key + + +def test_version_sort_key_orders_by_year_then_month(): + versions = ["01_2024", "12_2023", "06_2024", "03_2025"] + assert sorted(versions, key=clinvar_version_sort_key, reverse=True) == [ + "03_2025", + "06_2024", + "01_2024", + "12_2023", + ] + + +def test_version_sort_key_parses_month_and_year(): + # (year, month) — a later month within the same year sorts higher. + assert clinvar_version_sort_key("06_2024") == (2024, 6) + assert clinvar_version_sort_key("11_2024") > clinvar_version_sort_key("06_2024") + + +def test_version_sort_key_unparseable_sorts_to_bottom(): + # Malformed versions collapse to (0, 0) rather than raising, so they sink under any real release. + assert clinvar_version_sort_key("garbage") == (0, 0) + assert clinvar_version_sort_key("2024") == (0, 0) + assert clinvar_version_sort_key("13_20_2024") == (0, 0) + assert clinvar_version_sort_key("01_2020") > clinvar_version_sort_key("garbage") diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index 14dde9527..b3dc62c28 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -1,26 +1,28 @@ # ruff: noqa: E402 -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest - -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from sqlalchemy import select pyathena = pytest.importorskip("pyathena") fastapi = pytest.importorskip("fastapi") from mavedb.lib.gnomad import ( + GnomadLinkVerdict, allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, + normalize_caid, ) +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, - TEST_MINIMAL_MAPPED_VARIANT, + TEST_MAVEDB_ATHENA_ROW, ) ### Tests for gnomad_identifier function ### @@ -77,6 +79,23 @@ def test_gnomad_table_name_raises_if_env_not_set(): gnomad_table_name() +### Tests for normalize_caid function ### + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("CA025094", "CA25094"), # the #722 example: gnomAD dump drops the leading zero + ("CA000123", "CA123"), # multiple leading zeros collapse + ("CA341478553", "CA341478553"), # already unpadded — unchanged + ("CA0", "CA0"), # keep the final digit even if it is a zero + ("not-a-caid", "not-a-caid"), # unrecognized input is passed through + ], +) +def test_normalize_caid(raw, expected): + assert normalize_caid(raw) == expected + + ### Tests for allele_list_from_list_like_string function ### @@ -118,217 +137,247 @@ def test_allele_list_from_list_like_string_invalid_format_not_list(): # If the package is working correctly, this function should work as expected. -### Tests for link_gnomad_variants_to_mapped_variants function ### +### Tests for link_gnomad_variants_to_alleles function ### -def _verify_annotation_status(session, mapped_variants, expected_version): - annotations = session.query(VariantAnnotationStatus).all() - assert len(annotations) == len(mapped_variants) +def _make_allele(session, caid, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele carrying a CAID.""" + allele = Allele(vrs_digest=vrs_digest, level=level, clingen_allele_id=caid) + session.add(allele) + session.commit() + session.refresh(allele) + return allele - for mapped_variant, annotation in zip(mapped_variants, annotations): - assert annotation.variant_id == mapped_variant.variant_id - assert annotation.annotation_type == "gnomad_allele_frequency" - assert annotation.version == expected_version +def _make_gnomad_row(**overrides): + """Build an Athena-style gnomAD row Mock from the canonical test row, applying overrides. -def test_links_new_gnomad_variant_to_mapped_variant( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() + Mirrors the ``mocked_gnomad_variant_row`` fixture but takes overrides so a test can construct a + batch of distinct rows (different CAIDs, loci) in one call. + """ + row = Mock() + for key, value in {**TEST_MAVEDB_ATHENA_ROW, **overrides}.items(): + setattr(row, key, value) + return row - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 - session.commit() - session.refresh(mapped_variant) +def _live_links_for(session, allele_id): + return session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.current, + ) + ).all() - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] +def _assert_gnomad_variant_matches(gnomad_variant, **overrides): + expected = TEST_GNOMAD_VARIANT.copy() + expected.pop("creation_date") + expected.pop("modification_date") + expected.update(overrides) + for attr, value in expected.items(): + assert getattr(gnomad_variant, attr) == value - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} + session.commit() + + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant) -def test_can_link_gnomad_variants_with_none_type_faf_fields( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() + +def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max_gen_anc", None) mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max", None) with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - gnomad_variant_comparator["faf95_max"] = None - gnomad_variant_comparator["faf95_max_ancestry"] = None - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant, faf95_max=None, faf95_max_ancestry=None) - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) - -def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): +def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): gnomad_variant = GnomADVariant(**TEST_GNOMAD_VARIANT) - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) session.add(gnomad_variant) session.commit() + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") + # Reused the existing gnomAD variant rather than creating a second. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant_id == gnomad_variant.id - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) - - -def test_adding_existing_gnomad_variant_with_same_version_does_not_result_in_duplication( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() - - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 +def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): + """Supersede only on change: a second run with identical data writes nothing — one live link, + no retired rows, so the valid-time history records no spurious boundary. The second run still + *reports* the allele (verdict UNCHANGED), so the caller can mark it preexisting without re-querying + link state.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() + # Second run sees the live link already points to this gnomAD variant → unchanged, no DB write. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.UNCHANGED + } session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") + # One link, still live, never retired — the re-run did not churn the history. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 1 + assert all_links[0].valid_to is None + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - _verify_annotation_status(session, [mapped_variant, mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_variant_row): + """A new gnomAD version retires the prior link and installs the new one — exactly one live link + per allele (not one per version), with the old version preserved as a retired row.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() -def test_links_multiple_rows_and_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_version == "v2.new" + # Old-version link retired, not deleted; both gnomAD variant rows persist. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 2 + assert len([link for link in all_links if link.valid_to is not None]) == 1 + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + + +def test_same_version_different_identifier_supersedes_newest_wins(session, mocked_gnomad_variant_row): + """A CAID re-resolving to a different identifier within the same version is an anomaly: log and + supersede newest-wins rather than raise — one odd allele must not abort the batch.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + # Prior live link at the current version, but to a different identifier than the row resolves to. + stale = GnomADVariant( + db_name="gnomAD", + db_identifier="9-99999-C-T", + db_version=TEST_GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(stale) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=stale.id)) session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 2 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_identifier != "9-99999-C-T" # newest wins - assert len(mapped_variant1.gnomad_variants) == 1 - assert len(mapped_variant2.gnomad_variants) == 1 - for mv in [mapped_variant1, mapped_variant2]: - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mocked_gnomad_variant_row): + """A CAID shared by multiple alleles (cross-score-set dedup) fans the gnomAD variant to each.""" + allele1 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + allele2 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-2", level="cdna") + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele1.id: GnomadLinkVerdict.CREATED, allele2.id: GnomadLinkVerdict.CREATED} + session.commit() -def test_returns_zero_when_no_mapped_variants(session, mocked_gnomad_variant_row): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 0 - - _verify_annotation_status(session, [], TEST_GNOMAD_DATA_VERSION) - + for allele in (allele1, allele2): + assert len(_live_links_for(session, allele.id)) == 1 + # Both links point at the single get-or-created gnomAD variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 -def test_only_current_flag_filters_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gnomad_variant_row): + """The gnomAD dump records CAIDs without leading zeros (#722). An allele stored with the + zero-padded CAID must still match the dump's stripped form across the join.""" + allele = _make_allele(session, "CA025094", vrs_digest="vrs-1") + mocked_gnomad_variant_row.caid = "CA25094" # dump form: leading zero stripped with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") + assert len(_live_links_for(session, allele.id)) == 1 - assert len(mapped_variant1.gnomad_variants) == 0 - assert len(mapped_variant2.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant2.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - _verify_annotation_status(session, [mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_links_a_batch_of_rows_in_one_pass(session): + """Multiple rows in one call are resolved by a single batched allele lookup, then grouped by + normalized CAID. Exercises everything the batching must preserve at once: distinct CAIDs each + reach their own allele, a zero-padded stored CAID still matches a dump-stripped row (#722), and a + CAID shared across alleles fans its variant out to each — all without a per-row query.""" + # Row A: allele stored zero-padded, dump delivers the stripped form. + allele_a = _make_allele(session, "CA025094", vrs_digest="vrs-a") + row_a = _make_gnomad_row(caid="CA25094") + row_a.__setattr__("locus.position", "87961093") # identifier 10-87961093-A-G - -def test_only_current_flag_is_false_operates_on_all_variants( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() + # Row B: a different CAID at a different locus, shared by two alleles (cross-score-set dedup). + allele_b1 = _make_allele(session, "CA341478553", vrs_digest="vrs-b1") + allele_b2 = _make_allele(session, "CA341478553", vrs_digest="vrs-b2", level="cdna") + row_b = _make_gnomad_row(caid="CA341478553") + row_b.__setattr__("locus.position", "87961094") # identifier 10-87961094-A-G with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row], False) - assert result == 2 + result = link_gnomad_variants_to_alleles(session, [row_a, row_b]) + assert result == { + allele_a.id: GnomadLinkVerdict.CREATED, + allele_b1.id: GnomadLinkVerdict.CREATED, + allele_b2.id: GnomadLinkVerdict.CREATED, + } session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - for mv in [mapped_variant1, mapped_variant2]: - assert len(mv.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) + # Each allele ends with exactly one live link. + for allele in (allele_a, allele_b1, allele_b2): + assert len(_live_links_for(session, allele.id)) == 1 + + # Two distinct gnomAD variants (one per row's identifier); B's two alleles share row B's variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + b1_variant = _live_links_for(session, allele_b1.id)[0].gnomad_variant_id + b2_variant = _live_links_for(session, allele_b2.id)[0].gnomad_variant_id + a_variant = _live_links_for(session, allele_a.id)[0].gnomad_variant_id + assert b1_variant == b2_variant + assert a_variant != b1_variant + + +def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {} + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 + # No gnomAD variant is created when nothing matches the CAID. + assert len(session.scalars(select(GnomADVariant)).all()) == 0 diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py new file mode 100644 index 000000000..5ecb40e9d --- /dev/null +++ b/tests/lib/test_hgvs.py @@ -0,0 +1,198 @@ +import pytest + +from mavedb.lib.hgvs import ( + SequenceBlock, + extract_accession, + join_cis_phased_hgvs, + parse_simple_nucleotide_substitution, + parse_simple_protein_substitution, + parse_simple_substitution, + split_cis_phased_hgvs, + strip_protein_prediction_parens, +) + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # auto-detects the level from the disjoint coordinate prefix + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + # non-placeable in either grammar + ("c.122-6T>A", None), + ("c.[197A>G;472T>C]", None), + (None, None), + ], +) +def test_parse_simple_substitution(hgvs, expected): + assert parse_simple_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified and bare coding substitutions + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("c.5A>G", SequenceBlock(position=5, ref="A", alt="G")), + # genomic and non-coding levels parse the same way + ("NC_000001.11:g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + ("n.42C>T", SequenceBlock(position=42, ref="C", alt="T")), + # lowercase nucleotides are tolerated + ("c.5a>g", SequenceBlock(position=5, ref="a", alt="g")), + # non-placeable: UTR/intron positions, multivariant, indels, non-substitutions, empty + ("c.*123A>G", None), + ("c.-12A>G", None), + ("c.12+3A>G", None), + ("NM_000546.6:c.[197A>G;472T>C]", None), + ("c.76_78del", None), + ("p.Ala406Thr", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_nucleotide_substitution(hgvs, expected): + assert parse_simple_nucleotide_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified, bare, and prediction-wrapped substitutions + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("NP_000537.3:p.(Ala406Thr)", SequenceBlock(position=406, ref="Ala", alt="Thr")), + # synonymous, stop, and deletion tokens are preserved as the raw alt + ("p.Ala406=", SequenceBlock(position=406, ref="Ala", alt="=")), + ("p.Tyr745*", SequenceBlock(position=745, ref="Tyr", alt="*")), + ("p.Ala406-", SequenceBlock(position=406, ref="Ala", alt="-")), + # non-placeable: multivariant, frameshift, single-letter codes, empty + ("p.[Ala406Thr;Gly12Cys]", None), + ("p.Arg97fs", None), + ("p.A406T", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_protein_substitution(hgvs, expected): + assert parse_simple_protein_substitution(hgvs) == expected + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NP_000456.2:p.(Ala222Val)", "NP_000456.2:p.Ala222Val"), + ("NP_000456.2:p.(Tyr745Ter)", "NP_000456.2:p.Tyr745Ter"), + ("NP_000456.2:p.(Ala222_Val225del)", "NP_000456.2:p.Ala222_Val225del"), + # no prediction parens -> unchanged + ("NP_000456.2:p.Ala222Val", "NP_000456.2:p.Ala222Val"), + ("NM_000001.1:c.5A>G", "NM_000001.1:c.5A>G"), + ], +) +def test_strip_protein_prediction_parens(hgvs, expected): + assert strip_protein_prediction_parens(hgvs) == expected + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000001.11:g.1000A>G", "NC_000001.11"), + ("NM_000001.1:c.5A>G", "NM_000001.1"), + ("NC_000001.11:g.[1000A>G;1002T>C]", "NC_000001.11"), + ("no-colon-here", ""), + ("", ""), + ], +) +def test_extract_accession(hgvs, expected): + assert extract_accession(hgvs) == expected + + +def test_split_cis_phased_hgvs_passes_through_single_variant(): + assert split_cis_phased_hgvs("NC_000001.11:g.1000A>G") == ["NC_000001.11:g.1000A>G"] + + +def test_split_cis_phased_hgvs_splits_bracketed_genomic_expression(): + # Non-adjacent codon components are emitted as one bracketed genomic expression; each + # component must regain the accession and prefix to be VRS-translatable on its own. + assert split_cis_phased_hgvs("NC_000001.11:g.[1000A>G;1002T>C]") == [ + "NC_000001.11:g.1000A>G", + "NC_000001.11:g.1002T>C", + ] + + +def test_split_cis_phased_hgvs_handles_three_components_and_coding_prefix(): + assert split_cis_phased_hgvs("NM_000001.1:c.[1A>G;3T>C;5G>A]") == [ + "NM_000001.1:c.1A>G", + "NM_000001.1:c.3T>C", + "NM_000001.1:c.5G>A", + ] + + +def test_join_cis_phased_hgvs_passes_through_single_component(): + assert join_cis_phased_hgvs(["NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.1000A>G" + + +def test_join_cis_phased_hgvs_combines_genomic_components(): + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +@pytest.mark.parametrize( + "expression", + [ + "NC_000001.11:g.[1000A>G;1002T>C]", + "NM_000001.1:c.[1A>G;3T>C;5G>A]", + ], +) +def test_join_is_inverse_of_split(expression): + assert join_cis_phased_hgvs(split_cis_phased_hgvs(expression)) == expression + + +def test_join_cis_phased_hgvs_returns_none_for_empty(): + assert join_cis_phased_hgvs([]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_accessions(): + # Members on different sequences are not one cis-phased block. + assert join_cis_phased_hgvs(["NC_000001.11:g.1A>G", "NC_000002.12:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): + assert join_cis_phased_hgvs(["NM_000001.1:c.1A>G", "NM_000001.1:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): + assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None + + +def test_split_cis_phased_hgvs_passes_through_bracketed_without_accession(): + # Bracketed but accession-less input is not a cis-phased multivariant we can qualify; it must + # degrade to a single-element list rather than raising on the missing ":". + assert split_cis_phased_hgvs("g.[1000A>G;1002T>C]") == ["g.[1000A>G;1002T>C]"] + + +def test_join_cis_phased_hgvs_orders_components_by_position(): + # Out-of-order members are emitted in coordinate order. + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +def test_join_cis_phased_hgvs_is_order_independent(): + # The same set of members yields the same string regardless of input ordering (the VRS block + # digest is order-independent; the exported HGVS string must be too). + forward = join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) + reverse = join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) + assert forward == reverse + + +def test_join_cis_phased_hgvs_orders_protein_components_by_position(): + # The first integer is the position for protein forms too (Arg123Gly), not just genomic. + assert ( + join_cis_phased_hgvs(["NP_000001.1:p.Arg223Gly", "NP_000001.1:p.Ala12Val"]) + == "NP_000001.1:p.[Ala12Val;Arg223Gly]" + ) diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py new file mode 100644 index 000000000..6ac84bb12 --- /dev/null +++ b/tests/lib/test_score_set_variants.py @@ -0,0 +1,309 @@ +# ruff: noqa: E402 +"""Integration tests for the lean whole-set view assembly (``lib/score_set_variants.py``). + +These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped +assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the +authoritative allele; each HGVS string always rides even when it is not a placeable simple +substitution; unmapped variants are retained with null mapped fields; ``as_of`` reconstructs the +annotation layer while the immutable submitted HGVS and score stay put; and the set is ordered by +variant number. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.score_set_variants import HgvsField, MappedTriple, get_lean_score_set_variants +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock (mirrors tests/lib/test_alleles.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=None, hgvs_splice=None): + data = {"score_data": {"score": score}} if score is not None else TEST_MINIMAL_VARIANT["data"] + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data, + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + hgvs_splice=hgvs_splice, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_c=None, hgvs_g=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped={"type": "Allele"}, + clingen_allele_id=clingen_allele_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, valid_from=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=True, projection_group=None, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + valid_from=valid_from, + ) + session.add(link) + session.commit() + return link + + +def _consequence(session, allele, value): + row = VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + session.add(row) + session.commit() + return row + + +@pytest.mark.integration +def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant; the mapped triple carries the measured cdna slot + (from the record), its projection_group genomic sibling, and the protein apex; digest/ClinGen + id/consequence off the authoritative allele. Each parseable string gets its block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is a member of no pair (group None). + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, is_authoritative=False, projection_group=0) + _link(session, record, protein, is_authoritative=False) + _consequence(session, measured, "missense_variant") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == -2.3 + assert record_out.consequence == "missense_variant" + assert record_out.clingen_allele_id == "CA123" + assert record_out.assay_level_digest == "cdna-digest" # the measured (authoritative) allele + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") + assert record_out.hgvs_splice is None + # assayLevel names the measured slot; the triple fills all three levels. + assert record_out.assay_level == "cdna" + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_nucleotide_assay_measured_at_genomic(session, setup_lib_db_with_score_set): + """Genomic assay: the measured slot is genomic (from the record); the cdna slot — the level-invariant + search key — comes off the projection_group sibling even though the assay was measured at genomic.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA9") + cdna = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, cdna, is_authoritative=False, projection_group=0) + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "genomic" + assert record_out.assay_level_digest == "gen-digest" + # Measured slot is genomic (from the record); cdna (search key) comes off the sibling allele. + assert record_out.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_score_set): + """Protein assay: the measured slot *is* protein, so only the protein slot is populated; the cdna and + genomic slots stay null (the c/g fan-out is ambiguous — no canonical pick is fabricated).""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele( + session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + _link(session, record, measured, is_authoritative=True) + # A candidate fan-out member (cdna) with its own projection pair group — the protein authoritative link is + # in no group, so this candidate must NOT be pulled into the canonical triple. + candidate = _allele(session, "cand-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + _link(session, record, candidate, is_authoritative=False, projection_group=0) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "protein" + assert record_out.assay_level_digest == "prot-digest" + assert record_out.clingen_allele_id == "PA9" + # Only the protein slot; cdna/genomic stay null despite the candidate fan-out being linked. + assert record_out.mapped == MappedTriple( + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + ) + + +@pytest.mark.integration +def test_unpopulated_projection_group_degrades_to_a_null_sibling(session, setup_lib_db_with_score_set): + """Pre-reverse-translation data (no projection_group on the authoritative link): the sibling join + returns nothing, so the other nucleotide slot stays null — today's behavior — while the measured and + protein slots still populate. Still one row per variant.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # projection_group defaults to None + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "cdna" + # Measured cdna + protein apex populate; genomic is null (no recorded sibling — graceful degradation). + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_score_set): + """The canonical string is always present; a splice/multivariant expression simply carries no block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="NM_000546.6:c.[197A>G;472T>C]", hgvs_splice="c.122-6T>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="c.76_78del") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + # Multivariant, intronic, and indel expressions are all string-only (no position/ref/alt block). + assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") + assert record_out.hgvs_nt.position is None + assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") + # The measured cdna slot carries the (unparseable) indel string alone. + assert record_out.mapped.cdna == HgvsField(hgvs="c.76_78del") + + +@pytest.mark.integration +def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_score_set): + """A variant with no live mapping record keeps its submitted HGVS + score but null mapped fields.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.5, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == 0.5 + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.consequence is None + assert record_out.clingen_allele_id is None + assert record_out.assay_level_digest is None + assert record_out.assay_level is None + assert record_out.mapped == MappedTriple() # empty triple — no slots populated + + +@pytest.mark.integration +def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): + """A mapped variant with no live VEP consequence has consequence None but keeps its digest + mapped HGVS.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.consequence is None + assert record_out.assay_level_digest == "cdna-digest" + assert record_out.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + + +@pytest.mark.integration +def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): + """Records come back ordered by the integer after '#', not insertion or lexical order.""" + score_set = setup_lib_db_with_score_set + for suffix in (10, 2, 1): + _variant(session, score_set, suffix, score=float(suffix)) + + records = get_lean_score_set_variants(session, score_set) + + assert [r.variant_urn for r in records] == [f"{score_set.urn}#{n}" for n in (1, 2, 10)] + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_with_score_set): + """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at + a past instant, while the immutable submitted HGVS and score are unaffected.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + + old_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + old_allele = _allele(session, "old-digest", level="cdna") + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + + new_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.500A>T", valid_from=T1) + new_allele = _allele(session, "new-digest", level="cdna") + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + session.commit() + + [current] = get_lean_score_set_variants(session, score_set) + assert current.assay_level_digest == "new-digest" + assert current.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + + [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) + assert historical.assay_level_digest == "old-digest" + assert historical.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # Submitted HGVS and score are immutable — unaffected by as_of. + assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert historical.score == -2.3 + + # Before anything was mapped, the variant still appears with null mapped fields. + [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) + assert pre_mapping.assay_level_digest is None + assert pre_mapping.assay_level is None + assert pre_mapping.mapped == MappedTriple() diff --git a/tests/lib/test_term_systems.py b/tests/lib/test_term_systems.py new file mode 100644 index 000000000..962ce324e --- /dev/null +++ b/tests/lib/test_term_systems.py @@ -0,0 +1,14 @@ +"""Tests for the shared (system, prefix) -> Coding helper.""" + +import pytest + +from mavedb.lib.term_systems import coding + + +@pytest.mark.unit +def test_coding_builds_id_from_prefix_and_code(): + result = coding(("https://example.org/system", "ex"), "some_code") + + assert result.id == "ex:some_code" + assert result.code.root == "some_code" + assert result.system == "https://example.org/system" diff --git a/tests/lib/test_variant_annotations_script.py b/tests/lib/test_variant_annotations_script.py new file mode 100644 index 000000000..494bce22b --- /dev/null +++ b/tests/lib/test_variant_annotations_script.py @@ -0,0 +1,89 @@ +# ruff: noqa: E402 +"""Tests for the variant_annotations CLI's current_annotation_summary resolution. + +The score set is the entry point, but status is resolved per-allele through the live mapping links — +so a shared allele's status counts even when a *different* score set's run produced it. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.scripts.variant_annotations import current_annotation_summary +from tests.helpers.constants import TEST_MINIMAL_VARIANT + + +def _variant_mapped_to_allele(session, score_set, allele): + """A variant in the score set with a live mapping record + live authoritative link to ``allele``.""" + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + session.add(variant) + session.commit() + + record = MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0") + session.add(record) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=True)) + session.commit() + return variant + + +def test_summary_counts_allele_status_from_another_score_sets_run(session, setup_lib_db_with_score_set, job_run): + """An allele-subject status produced by a *different* score set's run (or none) still counts for a + score set whose variant currently maps to that shared allele.""" + score_set = setup_lib_db_with_score_set + allele = Allele( + vrs_digest="summary-shared", level="genomic", clingen_allele_id="CA1", post_mapped={"type": "Allele"} + ) + session.add(allele) + session.commit() + _variant_mapped_to_allele(session, score_set, allele) + + # Event written with no owning score set (e.g. a different score set's run touched the shared allele). + session.add( + AnnotationEvent( + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="created", + job_run_id=job_run.id, + score_set_id=None, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.CLINGEN_ALLELE_ID.value, Disposition.PRESENT.value)) == 1 + + +def test_summary_includes_variant_subject_status(session, setup_lib_db_with_score_set, job_run): + """Variant-subject types (mapping/RT/LDH) are counted per variant, off the score set's variants.""" + score_set = setup_lib_db_with_score_set + allele = Allele(vrs_digest="summary-vs", level="genomic", post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + variant = _variant_mapped_to_allele(session, score_set, allele) + + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.VRS_MAPPING.value, Disposition.PRESENT.value)) == 1 diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py new file mode 100644 index 000000000..8950dd6ef --- /dev/null +++ b/tests/lib/test_variant_detail.py @@ -0,0 +1,353 @@ +# ruff: noqa: E402 +"""Integration tests for the variant-detail envelope assembly (``lib/variant_detail.py``). + +These pin the composition: the flat assay fields (level, target/reference HGVS, digest, ClinGen id) +off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off +the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications +(primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / +``superseded_by_score_set`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +we assert they are wired in and keyed correctly. +""" + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.variant_detail import get_variant_detail +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT, TEST_USER + +# A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this, so it must parse. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_g=None, hgvs_c=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped=_post_mapped(), + clingen_allele_id=clingen_allele_id, + hgvs_g=hgvs_g, + hgvs_c=hgvs_c, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False, projection_group=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + ) + session.add(link) + session.commit() + return link + + +def _vep(session, allele, value): + session.add( + VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + ) + session.commit() + + +def _calibration(session, score_set, *, primary, classifications): + """A calibration with functional classifications; each ``classifications`` entry is + ``(functional, [variants...])`` and its m2m membership is set to those variants.""" + user = session.query(User).filter(User.username == TEST_USER["username"]).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + title=f"cal-{'primary' if primary else 'secondary'}", + primary=primary, + private=False, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + for functional, variants in classifications: + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, label=functional, functional_classification=functional + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +@pytest.mark.integration +def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): + """Mode 1 (coding measured): flat assay fields off the record + authoritative allele, spec-pure + Cat-VRS in projection mode, and a digest-keyed annotation map covering the linked alleles.""" + score_set = setup_lib_db_with_score_set + variant = _variant( + session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" + ) + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + # A synonymous cousin: an nt encoder of the same protein consequence in a *different* projection group + # — a distinct, unmeasured variant carried by the reverse-translation fan. + cousin = _allele(session, "cousin-digest", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is in no pair (group None); the cousin is in its own group. + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, projection_group=0) + _link(session, record, protein) + _link(session, record, cousin, projection_group=1) + _vep(session, measured, "missense_variant") + + detail = get_variant_detail(session, variant) + + assert detail.urn == variant.urn + assert detail.scores == {"score": -2.3} + assert detail.assay_level == "cdna" + assert detail.target_hgvs == "c.1216G>A" # submitted, target frame + assert detail.reference_hgvs == "NM_000546.6:c.1216G>A" # mapped, reference frame + assert detail.assay_level_digest == "cdna-digest" + assert detail.clingen_allele_id == "CA123" + assert detail.mode == "projection" + assert detail.molecular_representation is not None + assert detail.molecular_representation["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar — every linked allele is a + # member, and vice versa (the cousin included). + assert len(detail.molecular_representation["members"]) == len(detail.alleles) + # The alleles sidecar carries one identity per linked allele, keyed by digest: level + + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + + # derivation + projection_of. + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} + measured_identity = detail.alleles["cdna-digest"] + assert measured_identity.level == "cdna" + assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c + assert measured_identity.clingen_allele_id == "CA123" + assert measured_identity.relation is None # the defining allele has no relation to itself + # The measured allele is the focus; derivation describes the *other* members relative to it, so the + # focus carries none. It pairs with its genomic projection sibling. + assert measured_identity.is_focus is True + assert measured_identity.derivation is None + assert measured_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.is_focus is False + # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. + assert genomic_identity.level == "genomic" + assert genomic_identity.derivation == "projection" + assert genomic_identity.projection_of == "cdna-digest" + protein_identity = detail.alleles["prot-digest"] + assert protein_identity.level == "protein" + assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p + # The protein member is a translation of the measured coding allele (member -> defining)... + assert protein_identity.relation == "translation_of" + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) with no + # projection sibling (the apex is in no c/g pair). + assert protein_identity.derivation == "projection" + assert protein_identity.projection_of is None + # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and + # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured + # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with + # nothing (its group has a single member). + cousin_identity = detail.alleles["cousin-digest"] + assert cousin_identity.level == "cdna" + assert cousin_identity.relation == "co_encodes" + assert cousin_identity.derivation == "convergent" + assert cousin_identity.projection_of is None + # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} + assert detail.annotations["cdna-digest"].vep is not None + assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" + assert detail.is_current is True + assert detail.superseded_by_score_set is None + + +@pytest.mark.integration +def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, the nt + members `encode` the defining protein allele, and the fanned-out c/g candidates are labelled + `candidate` (ambiguous) while still pairing to each other via projection_of. The protein apex is in + no pair (group None). The `relation` (structural) and `derivation` (provenance) axes are independent: + the coding candidate is `encodes` + `candidate` here, versus `translation_of` + `projection` for the + protein member of a nucleotide assay.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") + coding = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True) # protein apex — group None + # One reverse-translation projection pair: a coding candidate paired with its genomic projection. + _link(session, record, coding, projection_group=0) + _link(session, record, genomic, projection_group=0) + + detail = get_variant_detail(session, variant) + + assert detail.assay_level == "protein" + assert detail.target_hgvs == "p.Ala406Thr" # protein assay -> submitted protein string + assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" + assert detail.assay_level_digest == "prot-digest" + assert detail.mode == "reverse_translation" + # Defining protein allele is the focus (no relation/derivation to itself), and is in no c/g pair. + prot = detail.alleles["prot-digest"] + assert prot.level == "protein" + assert prot.relation is None + assert prot.is_focus is True + assert prot.derivation is None + assert prot.projection_of is None + # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` + # (provenance) — the two axes disagree by design — and pairs to its genomic projection. + coding_identity = detail.alleles["cdna-digest"] + assert coding_identity.relation == "encodes" + assert coding_identity.derivation == "candidate" + assert coding_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.relation == "encodes" + assert genomic_identity.derivation == "candidate" + assert genomic_identity.projection_of == "cdna-digest" + + +@pytest.mark.integration +def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): + """Links written before reverse translation ran carry a NULL projection_group: focus/derivation are + still computed (is_focus from is_authoritative; projection from the assay level), but projection_of + degrades to null — nothing pairs until the data is re-processed.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # no projection_group (pre-RT) + _link(session, record, protein) + + detail = get_variant_detail(session, variant) + + assert detail.alleles["cdna-digest"].is_focus is True + assert detail.alleles["cdna-digest"].derivation is None + assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling + assert detail.alleles["prot-digest"].derivation == "projection" + assert detail.alleles["prot-digest"].projection_of is None + + +@pytest.mark.integration +def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_score_set): + """No live record: flat mapped fields and Cat-VRS are null, the annotation map is empty, and the + variant reads as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + + detail = get_variant_detail(session, variant) + + assert detail.scores == {"score": 0.5} + assert detail.assay_level is None + assert detail.target_hgvs == "c.1A>T" # submitted still surfaces (no assay level -> nucleotide) + assert detail.reference_hgvs is None + assert detail.assay_level_digest is None + assert detail.clingen_allele_id is None + assert detail.molecular_representation is None + assert detail.mode is None + assert detail.alleles == {} + assert detail.annotations == {} + assert detail.is_current is True + + +@pytest.mark.integration +def test_classifications_are_per_calibration_primary_first(session, setup_lib_db_with_score_set): + """A variant carries one classification per calibration that classifies it; the primary sorts first.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + + detail = get_variant_detail(session, variant) + + assert [(c.primary, c.classification.functional_classification.value) for c in detail.classifications] == [ + (True, "abnormal"), + (False, "normal"), + ] + + +@pytest.mark.integration +def test_visible_calibration_ids_filters_classifications(session, setup_lib_db_with_score_set): + """Only classifications from calibrations the caller resolved as visible are returned.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + visible = _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + + detail = get_variant_detail(session, variant, visible_calibration_ids={visible.id}) + + assert [c.calibration_id for c in detail.classifications] == [visible.id] + + +@pytest.mark.integration +def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set): + """When a superseding version is passed in, the variant reports it rather than reading as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + newer = SimpleNamespace(urn="urn:mavedb:00000001-a-2") # only .urn is read + + detail = get_variant_detail(session, variant, superseding_score_set=newer) + + assert detail.is_current is False + assert detail.superseded_by_score_set == "urn:mavedb:00000001-a-2" diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index f01650a05..1ef1e1a58 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -6,6 +6,7 @@ hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, + score_from_variant_data, ) from tests.helpers.constants import ( TEST_GA4GH_DIGEST, @@ -17,6 +18,28 @@ TEST_VALID_POST_MAPPED_VRS_HAPLOTYPE, ) +### Tests for score_from_variant_data function ### + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"score_data": {"score": -2.3}}, -2.3), + ({"score_data": {"score": 0}}, 0.0), # a real 0.0 score is not "missing" + ({"score_data": {"score": "1.5"}}, 1.5), # numeric strings coerce + ({"score_data": {"score": None}}, None), # explicit NA + ({"score_data": {}}, None), # no score key + ({"count_data": {"c": 1}}, None), # no score_data block + ({"score_data": {"score": True}}, None), # a JSON bool is not a score + ({"score_data": {"score": "NA"}}, None), # non-numeric string + ({}, None), + (None, None), + ], +) +def test_score_from_variant_data(data, expected): + assert score_from_variant_data(data) == expected + + ### Tests for hgvs_from_vrs_allele function ### @@ -48,6 +71,18 @@ def test_get_hgvs_from_post_mapped_cis_phased_block(): assert result is None +def test_get_hgvs_from_post_mapped_cis_phased_block_combine_cis(): + # combine_cis collapses the cis-phased members into one bracketed expression. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, combine_cis=True) + assert result == "NM_003345:p.[Asp5Phe;Asp5Phe]" + + +def test_get_hgvs_from_post_mapped_single_allele_combine_cis_is_unbracketed(): + # A single-variant post-mapped allele is unaffected by combine_cis. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, combine_cis=True) + assert result == TEST_HGVS_IDENTIFIER + + def test_get_hgvs_from_post_mapped_single_allele_vrs_1(): with pytest.raises(ValueError): get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X) @@ -73,6 +108,26 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) +def test_hgvs_from_vrs_allele_null_or_empty_expressions(): + # A VRS allele may carry `expressions: null` or `[]` — that is "no HGVS", not a crash. + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": None}) is None + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": []}) is None + + +def test_get_hgvs_from_post_mapped_member_without_expression(): + # Regression: a cis-phased block member whose `expressions` is null must yield None, not raise + # `TypeError: 'NoneType' object is not subscriptable` (which previously killed the CAR job). + block = { + "type": "CisPhasedBlock", + "members": [ + {"type": "Allele", "expressions": [{"value": "NM_003345:p.Asp5Phe"}]}, + {"type": "Allele", "expressions": None}, + ], + } + assert get_hgvs_from_post_mapped(block) is None + assert get_hgvs_from_post_mapped(block, combine_cis=True) is None + + ### Tests for get_id_from_post_mapped function ### diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 2e9e1faef..bea141ab2 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -4,11 +4,21 @@ logic correctly handles the actual Ensembl REST API response shapes. """ +from datetime import date, timedelta from unittest.mock import MagicMock, patch import pytest +from sqlalchemy import select -from mavedb.lib.vep import get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + VepLinkVerdict, + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) +from mavedb.models.allele import Allele +from mavedb.models.vep_allele_consequence import VepAlleleConsequence def _mock_response(data) -> MagicMock: @@ -240,3 +250,158 @@ async def test_raises_if_more_than_200_variants(self): """Passing more than 200 HGVS strings raises ValueError before any HTTP call.""" with pytest.raises(ValueError, match="maximum of 200"): await get_functional_consequence(["NM_007294.4:c.1A>T"] * 201) + + +### Tests for get_ensembl_release function ### + + +@pytest.mark.asyncio +async def test_get_ensembl_release_returns_release_as_string(): + """The /info/software release integer is returned as a string for use as source_version.""" + with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response({"release": 116})): + assert await get_ensembl_release() == "116" + + +### Tests for link_vep_consequences_to_alleles function ### + + +def _make_allele(session, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele.""" + allele = Allele(vrs_digest=vrs_digest, level=level) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _live_rows_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + + +def _all_rows_for(session, allele_id): + return session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele_id)).all() + + +def test_link_vep_creates_new_consequence(session): + """A consequence for an allele with no live row creates a single live row and is reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.CREATED} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + assert live[0].access_date == date.today() + + +def test_link_vep_unchanged_bumps_version_and_date_in_place(session): + """Re-confirming an unchanged consequence at a new release advances source_version and access_date + in place — no supersede, no new valid-time boundary. The allele is reported UNCHANGED (status + preexisting) so the caller need not re-query consequence state.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + # One row, still live, never retired — version and access_date advanced in place. + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 1 + assert all_rows[0].valid_to is None + assert all_rows[0].source_version == "116" + assert all_rows[0].access_date == date.today() + + +def test_link_vep_changed_consequence_supersedes(session): + """A changed consequence retires the live row and inserts the successor — exactly one live row, + keyed on allele_id, with the old one preserved as retired history.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.CREATED} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + +def test_link_vep_none_leaves_live_row_untouched(session): + """A transient None result must not overwrite a held consequence: the live row is left intact + (value, version, and date). The held consequence is reported UNCHANGED (status preexisting) — the + allele still has a live consequence, it just was not re-confirmed this run.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + # Not re-confirmed -> neither version nor access_date advanced. + assert live[0].source_version == "115" + assert live[0].access_date == date.today() - timedelta(days=90) + + +def test_link_vep_none_with_no_live_row_writes_nothing(session): + """A None result for an allele with no live row writes nothing and leaves the allele out of the + verdict map (the caller reads that as a no-result and re-queries next run), mirroring gnomAD's + no-match handling.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {} + assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/lib/test_vrs.py b/tests/lib/test_vrs.py new file mode 100644 index 000000000..fae8dda67 --- /dev/null +++ b/tests/lib/test_vrs.py @@ -0,0 +1,96 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.vrs — deserialization of stored post_mapped dicts into GA4GH VRS objects.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from tests.helpers.constants import ( + TEST_SEQUENCE_LOCATION_ACCESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, +) + + +@pytest.mark.unit +class TestVariationExtractionUnit: + @pytest.mark.parametrize( + "variation_version", + [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], + ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], + ) + def test_vrs_object_from_post_mapped_variation(self, variation_version): + result = vrs_object_from_mapped_variant(variation_version).model_dump() + + assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION + assert result["location"]["start"] == 5 + assert result["location"]["end"] == 6 + + @pytest.mark.parametrize( + "allele_dict, expected_state_type, expected_state_fields", + [ + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + "LiteralSequenceExpression", + {"sequence": "F"}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + "ReferenceLengthExpression", + {"length": 5, "repeatSubunitLength": 5}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + "LengthExpression", + {"length": 5}, + ), + ], + ids=["lse_state", "rle_state", "length_expression_state"], + ) + def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): + result = vrs_object_from_mapped_variant(allele_dict).model_dump() + + assert result["state"]["type"] == expected_state_type + for k, v in expected_state_fields.items(): + assert result["state"][k] == v + + def test_unknown_allele_state_type_raises_value_error(self): + allele_dict = { + **TEST_VALID_POST_MAPPED_VRS_ALLELE, + "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, + } + + with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): + vrs_object_from_mapped_variant(allele_dict) + + @pytest.mark.parametrize( + "member_dict, expected_state_type", + [ + (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), + ], + ids=["lse_members", "rle_members", "length_expression_members"], + ) + def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): + mapping_results = {"type": "CisPhasedBlock", "members": [deepcopy(member_dict), deepcopy(member_dict)]} + + result = vrs_object_from_mapped_variant(mapping_results).model_dump() + + assert result["type"] == "CisPhasedBlock" + assert len(result["members"]) == 2 + assert result["members"][0]["state"]["type"] == expected_state_type + + +@pytest.mark.integration +class TestVrsIntegration: + def test_vrs_object_from_persisted_post_mapped(self, setup_lib_db_with_mapped_variant): + variation = vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + + assert variation is not None + assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py new file mode 100644 index 000000000..3b279560c --- /dev/null +++ b/tests/lib/test_vrs_utils.py @@ -0,0 +1,242 @@ +# ruff: noqa: E402 + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("ga4gh.vrs") + +from ga4gh.core.models import iriReference +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + Range, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + +from mavedb.lib import vrs_utils +from mavedb.lib.vrs_utils import ( + _rle_to_lse, + identify_variation, + normalize_and_identify, + translate_hgvs_to_variation, +) + +_SQ = "SQ." + "a" * 32 + +# A digest a reused ga4gh AlleleTranslator might leave on a component before it is +# re-identified — stale from the Merkle cache, or carried over from a sibling variant. +_STALE_ID = "ga4gh:VA." + "Z" * 32 + + +def _allele(start: int, alt: str) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + 1, + ), + state=LiteralSequenceExpression(sequence=alt), + ) + + +def _patch_component_translator(monkeypatch, alleles_by_hgvs: dict[str, Allele]) -> None: + """Stub the per-component single-allele translator that translate_hgvs_to_variation calls.""" + + def _fake(hgvs: str, translator=None) -> Allele: + return alleles_by_hgvs[hgvs] + + monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) + + +def _stub_normalize(monkeypatch) -> None: + """Make normalization a no-op so identification can be exercised without a seqrepo. + + Identification (the regression surface) runs for real; only the proxy-backed + normalize step is stubbed out. + """ + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: allele) + + +def _translator() -> SimpleNamespace: + return SimpleNamespace(data_proxy=object()) + + +def test_single_variant_returns_a_bare_allele(monkeypatch): + allele = _allele(1000, "G") + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert isinstance(result, Allele) + assert not isinstance(result, CisPhasedBlock) + + +def test_single_variant_is_reidentified_not_left_with_translator_digest(monkeypatch): + # ga4gh's translate_from (do_normalize=False) stamps id via plain ga4gh_identify on + # the reused translator; translate_hgvs_to_variation must re-identify so the persisted + # vrs_digest reflects current content rather than that carried-over value. + allele = _allele(1000, "G") + allele.id = _STALE_ID + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert result.id is not None + assert result.id != _STALE_ID + assert result.id.startswith("ga4gh:VA.") + + +def test_same_position_distinct_alts_get_distinct_digests(monkeypatch): + # Regression: NC_000002.12:g.214809459A>C and g.214809459A>T are different MaveDB + # variants that cannot encode the same codon, yet were merged onto one Allele row + # (and thus one coding equivalent like NM_000465.4:c.109_111delinsCGA). The single + # component returned by the reused translator carried a stale/shared digest, which + # the digest-keyed get_or_create_allele then deduplicated. Re-identification must + # give distinct content distinct digests so they never collide. + a_c = _allele(214809458, "C") + a_t = _allele(214809458, "T") + a_c.id = a_t.id = _STALE_ID # what the unfixed path would persist for both + _patch_component_translator( + monkeypatch, + { + "NC_000002.12:g.214809459A>C": a_c, + "NC_000002.12:g.214809459A>T": a_t, + }, + ) + _stub_normalize(monkeypatch) + translator = _translator() + + res_c = translate_hgvs_to_variation("NC_000002.12:g.214809459A>C", translator=translator) + res_t = translate_hgvs_to_variation("NC_000002.12:g.214809459A>T", translator=translator) + + assert res_c.id != _STALE_ID + assert res_t.id != _STALE_ID + assert res_c.id != res_t.id + + +def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + + assert isinstance(result, CisPhasedBlock) + assert len(result.members) == 2 + assert result.id is not None and result.id.startswith("ga4gh:CPB.") + + +def test_cis_phased_block_digest_is_order_independent(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) + + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=_translator()) + + # The same biological cis-phased set dedups to one row regardless of component ordering. + assert forward.id == reverse.id + + +def test_identify_variation_clears_stale_block_digest(): + block = CisPhasedBlock(members=[_allele(1000, "G"), _allele(1002, "C")]) + block.digest = "STALE" + + digest = identify_variation(block) + + assert digest != "STALE" + assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache + + +def _rle_allele(start: int, *, length: int, repeat_subunit_length: int) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + repeat_subunit_length, + ), + state=ReferenceLengthExpression(length=length, repeatSubunitLength=repeat_subunit_length), + ) + + +def test_rle_to_lse_tiles_repeat_subunit(): + # repeatSubunitLength=2 reads 2 bases from the proxy; length=4 tiles them out to "ACAC". + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = _rle_to_lse(rle, location, data_proxy) + + assert isinstance(result, LiteralSequenceExpression) + assert result.sequence.root == "ACAC" + + +def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): + # Regression: normalization can yield an RLE for an indel, but dcd_mapping stores LSE; the + # finalize step must coerce so the same variant doesn't hash to two digests (and duplicate). + rle = _rle_allele(10, length=2, repeat_subunit_length=2) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: rle) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = normalize_and_identify(rle, data_proxy=data_proxy) + + assert isinstance(result.state, LiteralSequenceExpression) + assert result.id is not None and result.id.startswith("ga4gh:VA.") + + +# The unions below (location.sequenceReference, location.start, rle.length) carry IRI-reference +# and Range variants that a fully-resolved indel allele never has. _rle_to_lse and the RLE branch +# of normalize_and_identify guard the inlined-and-integer contract with asserts; these tests pin +# that the guards fire rather than letting an IRI/Range slip into the digest computation as a +# silent AttributeError or wrong sequence read. +_NEVER_READ = SimpleNamespace(get_sequence=lambda identifier, start, end: pytest.fail("proxy read despite bad input")) + + +def test_rle_to_lse_rejects_iri_sequence_reference(): + location = SequenceLocation(sequenceReference=iriReference("seqref:unresolved"), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_start(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=Range([10, 12]), end=14) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_length(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=Range([2, 4]), repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_normalize_and_identify_rejects_iri_location_for_rle(monkeypatch): + # If normalization ever returned an RLE allele whose location is an unresolved IRI reference, + # _rle_to_lse could not read a sequence from it; the call-site assert must catch this before + # the digest is (mis)computed rather than crashing deeper with an AttributeError. + bad = Allele( + location=iriReference("loc:unresolved"), + state=ReferenceLengthExpression(length=2, repeatSubunitLength=2), + ) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: bad) + + with pytest.raises(AssertionError): + normalize_and_identify(bad, data_proxy=_NEVER_READ) diff --git a/tests/lib/workflow/test_kickoff.py b/tests/lib/workflow/test_kickoff.py new file mode 100644 index 000000000..9b98eaf45 --- /dev/null +++ b/tests/lib/workflow/test_kickoff.py @@ -0,0 +1,258 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import AsyncMock, Mock, patch + +from arq.jobs import Job as ArqJob +from sqlalchemy import select + +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.user import User +from tests.helpers.constants import TEST_USER + + +def _mock_entrypoint(*, id=42, job_function="start_pipeline", urn="urn:mavedb:job-fixed", retry_count=0): + return Mock(id=id, job_function=job_function, urn=urn, retry_count=retry_count) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetUnit: + """Unit tests with PipelineFactory and ARQ mocked out.""" + + @pytest.fixture(autouse=True) + def setup(self): + self.db = Mock() + self.redis = AsyncMock() + self.score_set = Mock(id=99, urn="urn:mavedb:00000001-a-1") + self.user = Mock(id=5) + self.pipeline = Mock(spec=Pipeline) + self.entrypoint = _mock_entrypoint() + + factory_patch = patch("mavedb.lib.workflow.kickoff.PipelineFactory") + self.factory_cls = factory_patch.start() + self.factory = self.factory_cls.return_value + self.factory.create_pipeline.return_value = (self.pipeline, self.entrypoint) + yield + factory_patch.stop() + + async def _call(self, **kwargs): + return await enqueue_pipeline_for_score_set( + self.db, self.redis, pipeline_name="test_pipeline", score_set=self.score_set, user=self.user, **kwargs + ) + + async def test_returns_pipeline_and_job_when_enqueued(self): + job = Mock(spec=ArqJob) + self.redis.enqueue_job.return_value = job + + result = await self._call() + + assert result == (self.pipeline, job) + self.redis.enqueue_job.assert_awaited_once_with( + self.entrypoint.job_function, + self.entrypoint.id, + _job_id=f"{self.entrypoint.urn}#{self.entrypoint.retry_count}", + ) + + async def test_returns_none_job_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + result = await self._call() + + assert result == (self.pipeline, None) + + async def test_propagates_key_error_for_unknown_pipeline_name(self): + self.factory.create_pipeline.side_effect = KeyError("unknown_pipeline") + + with pytest.raises(KeyError): + await self._call() + + async def test_discards_pipeline_and_reraises_when_enqueue_raises(self): + self.redis.enqueue_job.side_effect = ConnectionError("redis is unreachable") + + with pytest.raises(ConnectionError): + await self._call() + + self.factory.discard_pipeline.assert_called_once_with(self.pipeline) + + async def test_does_not_discard_pipeline_when_enqueue_succeeds(self): + self.redis.enqueue_job.return_value = Mock(spec=ArqJob) + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_does_not_discard_pipeline_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_default_correlation_id_built_from_pipeline_name_score_set_and_user(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value=None): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"].startswith(f"test_pipeline_{self.score_set.urn}_{self.user.id}_") + + async def test_correlation_id_from_context_used_when_present(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "context-correlation-id" + + async def test_extra_params_correlation_id_overrides_context(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call(extra_params={"correlation_id": "explicit-correlation-id"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "explicit-correlation-id" + + async def test_extra_params_merged_alongside_base_params(self): + await self._call(extra_params={"some_extra_param": "some_value"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["some_extra_param"] == "some_value" + assert params["score_set_id"] == self.score_set.id + assert params["updater_id"] == self.user.id + + async def test_create_pipeline_called_with_pipeline_name_and_user(self): + await self._call() + + call_kwargs = self.factory.create_pipeline.call_args.kwargs + assert call_kwargs["pipeline_name"] == "test_pipeline" + assert call_kwargs["creating_user"] is self.user + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetIntegration: + """Integration tests exercising the real PipelineFactory and a fakeredis-backed ARQ instance.""" + + @pytest.fixture(autouse=True) + def setup(self, session, setup_lib_db_with_score_set): + self.score_set = setup_lib_db_with_score_set + self.user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + async def test_creates_pipeline_and_job_run_records_and_enqueues_job( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, job = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert job is not None + assert session.execute(select(Pipeline).where(Pipeline.id == pipeline.id)).scalar_one() is pipeline + + job_run = session.execute( + select(JobRun).where(JobRun.pipeline_id == pipeline.id, JobRun.job_function == "process_data") + ).scalar_one() + assert job_run.job_params["required_param"] == "some_value" + + async def test_correlation_id_defaults_to_generated_value_without_request_context( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, _ = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline.correlation_id.startswith( + f"{sample_independent_pipeline_definition['name']}_{self.score_set.urn}_{self.user.id}_" + ) + + async def test_independent_calls_create_independent_pipelines_and_jobs( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline_one.id != pipeline_two.id + assert job_one is not None + assert job_two is not None + + async def test_returns_none_job_when_arq_job_id_collides( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with patch("mavedb.lib.workflow.kickoff.arq_job_id", return_value="fixed-dedupe-id"): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # ARQ deduplicates the second enqueue on the collided job id, but pipeline records are + # still created for both calls -- kickoff does not roll back on dedup. + assert job_one is not None + assert job_two is None + assert pipeline_one.id != pipeline_two.id + + async def test_unknown_pipeline_name_raises_key_error(self, session, arq_redis): + with pytest.raises(KeyError): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name="does_not_exist_pipeline", + score_set=self.score_set, + user=self.user, + ) + + async def test_discards_pipeline_records_when_enqueue_raises( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with ( + patch.object(arq_redis, "enqueue_job", AsyncMock(side_effect=ConnectionError("redis is unreachable"))), + pytest.raises(ConnectionError), + ): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # The pipeline (and its job runs) created before the failed enqueue must not be left behind. + assert session.execute(select(Pipeline)).scalars().all() == [] + assert session.execute(select(JobRun)).scalars().all() == [] diff --git a/tests/models/conftest.py b/tests/models/conftest.py new file mode 100644 index 000000000..5edfe06ee --- /dev/null +++ b/tests/models/conftest.py @@ -0,0 +1,80 @@ +import pytest + +from mavedb.models.enums import JobStatus +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.job_run import JobRun +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_EXPERIMENT, + TEST_EXPERIMENT_SET, + TEST_LICENSE, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, + TEST_USER, + VALID_EXPERIMENT_SET_URN, + VALID_EXPERIMENT_URN, + VALID_SCORE_SET_URN, +) + + +@pytest.fixture +def setup_lib_db_with_score_set(session, setup_lib_db): + """Build an experiment set, experiment, and score set on top of the base lib db (users/licenses).""" + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + experiment_set = ExperimentSet(**TEST_EXPERIMENT_SET, urn=VALID_EXPERIMENT_SET_URN) + experiment_set.created_by = user + experiment_set.modified_by = user + session.add(experiment_set) + session.commit() + session.refresh(experiment_set) + + experiment = Experiment(**TEST_EXPERIMENT, urn=VALID_EXPERIMENT_URN, experiment_set_id=experiment_set.id) + experiment.created_by = user + experiment.modified_by = user + session.add(experiment) + session.commit() + session.refresh(experiment) + + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, urn=VALID_SCORE_SET_URN, experiment_id=experiment.id, licence_id=TEST_LICENSE["id"] + ) + score_set.created_by = user + score_set.modified_by = user + session.add(score_set) + session.commit() + session.refresh(score_set) + + return score_set + + +@pytest.fixture +def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): + """Add a single variant to the score set, for variant-subject event tests.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#1", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + return variant + + +@pytest.fixture +def job_run(session): + """Create a persisted JobRun to anchor an event's provenance.""" + job = JobRun( + job_type="test_annotation_job", + job_function="test_function", + status=JobStatus.RUNNING, + ) + session.add(job) + session.commit() + session.refresh(job) + return job diff --git a/tests/models/test_annotation_event_model.py b/tests/models/test_annotation_event_model.py new file mode 100644 index 000000000..336536479 --- /dev/null +++ b/tests/models/test_annotation_event_model.py @@ -0,0 +1,103 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy.exc import IntegrityError + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +@pytest.fixture +def allele(session): + allele = Allele(vrs_digest="test-variant-event-allele-digest", level="genomic") + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _expect_rejected(session, event): + """Adding `event` must violate a CHECK constraint on flush.""" + session.add(event) + with pytest.raises(IntegrityError): + session.flush() + session.rollback() + + +class TestSubjectConstraint: + def test_variant_subject_event_inserts(self, session, setup_lib_db_with_variant, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + score_set_id=setup_lib_db_with_variant.score_set_id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.allele_id is None + + def test_allele_subject_event_inserts(self, session, allele, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason="no_record", + source_version="4.1.0", + job_run_id=job_run.id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.variant_id is None + + def test_variant_subject_type_with_allele_id_rejected(self, session, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_allele_subject_type_with_variant_id_rejected(self, session, setup_lib_db_with_variant): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="created", + ), + ) + + def test_neither_subject_set_rejected(self, session): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_both_subjects_set_rejected(self, session, setup_lib_db_with_variant, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) diff --git a/tests/models/test_annotation_event_view.py b/tests/models/test_annotation_event_view.py new file mode 100644 index 000000000..14c74f802 --- /dev/null +++ b/tests/models/test_annotation_event_view.py @@ -0,0 +1,147 @@ +# ruff: noqa: E402 +"""Tests for the v_current_annotation_events view (current event per subject + type).""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.annotation_event_view import CurrentAnnotationEventView +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +def _allele(session, digest, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _event(session, job_run, annotation_type, disposition, reason, *, allele_id=None, variant_id=None, **kw): + event = AnnotationEvent( + annotation_type=annotation_type, + allele_id=allele_id, + variant_id=variant_id, + disposition=disposition, + reason=reason, + job_run_id=job_run.id, + **kw, + ) + session.add(event) + session.commit() + session.refresh(event) + return event + + +def _view_rows(session, **filters): + stmt = select(CurrentAnnotationEventView) + for col, val in filters.items(): + stmt = stmt.where(getattr(CurrentAnnotationEventView, col) == val) + return list(session.scalars(stmt).all()) + + +def test_returns_only_latest_event_per_subject_and_type(session, job_run): + allele = _allele(session, "view-latest") + _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.ABSENT, "no_record", allele_id=allele.id + ) + latest = _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.PRESENT, "created", allele_id=allele.id + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].disposition == Disposition.PRESENT + + +def test_clinvar_is_multi_live_one_row_per_release(session, job_run): + allele = _allele(session, "view-clinvar") + e_2025 = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2025", + ) + _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2026", + ) + e_2026_latest = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "superseded", + allele_id=allele.id, + source_version="01_2026", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.CLINVAR_CONTROL.value) + + by_version = {r.source_version: r.id for r in rows} + assert by_version == {"01_2025": e_2025.id, "01_2026": e_2026_latest.id} + + +def test_non_clinvar_collapses_across_versions(session, job_run): + """gnomAD/VEP supersede to a single current state: a re-fetch at a new source_version yields one + row, not one per version (the CASE folds source_version in only for ClinVar).""" + allele = _allele(session, "view-gnomad-version") + _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="4.0.0", + ) + latest = _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "reconfirmed", + allele_id=allele.id, + source_version="4.1.0", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].source_version == "4.1.0" + + +def test_variant_subject_events_present_and_keyed_by_variant(session, setup_lib_db_with_variant, job_run): + variant = setup_lib_db_with_variant + _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.FAILED, "failed", variant_id=variant.id) + latest = _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.PRESENT, "mapped", variant_id=variant.id) + + rows = _view_rows(session, variant_id=variant.id, annotation_type=AnnotationType.VRS_MAPPING.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].allele_id is None + + +def test_distinct_subjects_yield_distinct_rows(session, job_run): + a1 = _allele(session, "view-a1") + a2 = _allele(session, "view-a2") + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.PRESENT, "created", allele_id=a1.id) + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.NOT_APPLICABLE, "no_hgvs", allele_id=a2.id) + + rows = _view_rows(session, annotation_type=AnnotationType.CLINGEN_ALLELE_ID.value) + + assert {r.allele_id for r in rows} == {a1.id, a2.id} diff --git a/tests/models/test_variant_annotation_view.py b/tests/models/test_variant_annotation_view.py new file mode 100644 index 000000000..d1c2f1d72 --- /dev/null +++ b/tests/models/test_variant_annotation_view.py @@ -0,0 +1,115 @@ +# ruff: noqa: E402 +"""Tests for the v_variant_annotations view after its rewrite onto the MappingRecord/Allele substrate. + +The view is a flat operator convenience projection: one row per (variant, current annotation_type). It +reconstructs the legacy ``mapped_variants`` row's flat ``hgvs_g``/``hgvs_c``/``hgvs_p`` triple by +pivoting the record's live allele links by level, and carries the single-valued ``clingen_allele_id`` +and VEP consequence from the record's authoritative allele. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_view import VariantAnnotationView +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +def _rows_for(session, variant_urn): + # Select explicit columns (not the whole entity): the view is keyless, so an all-null-identity row + # (an unmapped variant) would materialize as a single ``None`` entity. Column rows sidestep that. + stmt = select( + VariantAnnotationView.mapping_record_id, + VariantAnnotationView.hgvs_assay_level, + VariantAnnotationView.hgvs_g, + VariantAnnotationView.hgvs_c, + VariantAnnotationView.hgvs_p, + VariantAnnotationView.clingen_allele_id, + VariantAnnotationView.vep_functional_consequence, + ).where(VariantAnnotationView.variant_urn == variant_urn) + return list(session.execute(stmt).all()) + + +def test_reconstructs_flat_triple_and_authoritative_fields(session, setup_lib_db_with_variant): + """A cdna-assay record with genomic/cdna/protein alleles fills all three HGVS slots; the + authoritative (cdna) allele supplies clingen_allele_id and the VEP consequence.""" + variant = setup_lib_db_with_variant + record = seed_mapping_record( + session, + variant, + assay_level="cdna", + hgvs_assay_level="NM_000001.1:c.1A>T", + alleles=[ + AlleleSpec( + digest="d-cdna", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + hgvs_c="NM_000001.1:c.1A>T", + vep_consequence="missense_variant", + ), + AlleleSpec(digest="d-genomic", level="genomic", hgvs_g="NC_000001.11:g.100A>T"), + AlleleSpec(digest="d-protein", level="protein", hgvs_p="NP_000001.1:p.Met1Leu"), + ], + ) + + rows = _rows_for(session, variant.urn) + assert len(rows) == 1 + row = rows[0] + assert row.mapping_record_id == record.id + assert row.hgvs_assay_level == "NM_000001.1:c.1A>T" + assert row.hgvs_g == "NC_000001.11:g.100A>T" + assert row.hgvs_c == "NM_000001.1:c.1A>T" + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.clingen_allele_id == "CA123" + assert row.vep_functional_consequence == "missense_variant" + + +def test_protein_assay_leaves_nucleotide_slots_null(session, setup_lib_db_with_variant): + """A protein-assay record has only a protein allele: hgvs_p is filled, the nucleotide slots stay + null, and the authoritative protein allele still supplies clingen_allele_id.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="protein", + hgvs_assay_level="NP_000001.1:p.Met1Leu", + alleles=[ + AlleleSpec( + digest="d-prot-only", + level="protein", + is_authoritative=True, + clingen_allele_id="PA9", + hgvs_p="NP_000001.1:p.Met1Leu", + ), + ], + ) + + row = _rows_for(session, variant.urn)[0] + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.clingen_allele_id == "PA9" + + +def test_unmapped_variant_retained_with_null_mapping(session, setup_lib_db_with_score_set): + """The outer join keeps a variant with no live mapping record — it still appears with a null + mapping_record_id and empty mapped fields.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#7", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + row = _rows_for(session, variant.urn)[0] + assert row.mapping_record_id is None + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.hgvs_p is None + assert row.clingen_allele_id is None + assert row.vep_functional_consequence is None diff --git a/tests/routers/conftest.py b/tests/routers/conftest.py index ba34c5489..0971bde47 100644 --- a/tests/routers/conftest.py +++ b/tests/routers/conftest.py @@ -3,7 +3,7 @@ import pytest -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.enums.user_role import UserRole @@ -52,8 +52,8 @@ def setup_router_db(session): db.add(License(**TEST_INACTIVE_LICENSE)) db.add(License(**EXTRA_LICENSE)) db.add(Contributor(**EXTRA_CONTRIBUTOR)) - db.add(ClinicalControl(**TEST_CLINVAR_CONTROL)) - db.add(ClinicalControl(**TEST_GENERIC_CLINICAL_CONTROL)) + db.add(ClinvarControl(**TEST_CLINVAR_CONTROL)) + db.add(ClinvarControl(**TEST_GENERIC_CLINICAL_CONTROL)) db.add(GnomADVariant(**TEST_GNOMAD_VARIANT)) db.bulk_save_objects([ControlledKeyword(**keyword_obj) for keyword_obj in TEST_DB_KEYWORDS]) db.commit() diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py new file mode 100644 index 000000000..7ce80c0c9 --- /dev/null +++ b/tests/routers/test_allele.py @@ -0,0 +1,224 @@ +# ruff: noqa: E402 +"""Router tests for the allele-detail endpoints (``GET /alleles/{digest}`` and ``?clingenAlleleId=``). + +Exercise the HTTP surface end to end: the envelope serializes and validates, the equivalence class is +labelled relative to the focus, CAID fetch focuses the nt-canonical change, ``as_of`` is echoed and +reconstructs membership, an unknown id is a plain 404 — and, unlike ``GET /variants/{urn}``, the +endpoint is **public with no privacy gate**: an allele reachable only through a private score set is +still served. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.view_models.allele_detail import AlleleDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _ir(tag: str) -> str: + """A valid GA4GH IR (``ga4gh:VA.`` + exactly 32 base64url chars), tagged for readability.""" + return f"ga4gh:VA.{tag.ljust(32, '0')}" + + +CDNA = _ir("cdna") +GEN = _ir("gen") +PROT = _ir("prot") +COUSIN = _ir("cousin") + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed(session, variant_urn): + """Coding-measured record: authoritative cdna (+ClinGen CA1 +VEP), its genomic partner (also CA1), + the protein apex, and a synonymous cousin in its own projection group.""" + seed_mapping_record( + session, + variant_urn, + alleles=[ + AlleleSpec( + digest=CDNA, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest=GEN, + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec( + digest=PROT, + level="protein", + clingen_allele_id="PA1", + hgvs_p="NP_000537.3:p.Ala406Thr", + post_mapped=_post_mapped(), + ), + AlleleSpec( + digest=COUSIN, + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ], + assay_level="cdna", + ) + + +def test_get_allele_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + + assert body["digest"] == CDNA + assert body["level"] == "cdna" + assert body["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["clingenAlleleId"] == "CA1" + assert body["vrs"]["type"] == "Allele" + # Full equivalence class, keyed by digest, labelled relative to the queried allele (camelCase). + assert set(body["alleles"]) == {CDNA, GEN, PROT, COUSIN} + assert body["alleles"][CDNA]["isFocus"] is True + assert "derivation" not in body["alleles"][CDNA] # focus carries none; dropped by exclude_none + assert body["alleles"][GEN]["relation"] == "coordinate_representation_of" + assert body["alleles"][GEN]["derivation"] == "projection" + assert body["alleles"][PROT]["relation"] == "translation_of" + assert body["alleles"][COUSIN]["relation"] == "co_encodes" + assert body["alleles"][COUSIN]["derivation"] == "convergent" + # Annotations share keys with the class and join by digest. + assert set(body["annotations"]) == {CDNA, GEN, PROT, COUSIN} + assert body["annotations"][CDNA]["vep"]["consequence"] == "missense_variant" + # Measurement-agnostic: no variant-detail measurement fields. + assert "molecularRepresentation" not in body + assert "classifications" not in body + assert "isCurrent" not in body + assert response.headers["X-As-Of"] == "current" + + +def test_get_allele_detail_by_caid(client, session, data_provider, data_files, setup_router_db): + """CAID fetch on the overloaded `/alleles/{identifier}` path focuses the nt-canonical change (both + genomic + coding frames isFocus).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/CA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "CA1" + assert body["alleles"][CDNA]["isFocus"] is True + assert body["alleles"][GEN]["isFocus"] is True # the CAID's other frame + assert body["alleles"][COUSIN]["derivation"] == "convergent" + + +def test_get_allele_detail_by_paid(client, session, data_provider, data_files, setup_router_db): + """PAID fetch focuses the protein allele; its nucleotide equivalents surface as candidates — the + same `/alleles/{identifier}` path, no separate route or param.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/PA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "PA1" + assert body["alleles"][PROT]["isFocus"] is True + # Walking down from the protein is ambiguous: every nucleotide member is a candidate. + for nt in (CDNA, GEN, COUSIN): + assert body["alleles"][nt]["derivation"] == "candidate", nt + + +def test_get_allele_detail_unknown_digest_is_404(client, setup_router_db): + response = client.get(f"/api/v1/alleles/{_ir('nonexistent')}") + assert response.status_code == 404 + + +def test_get_allele_detail_unknown_caid_is_404(client, setup_router_db): + response = client.get("/api/v1/alleles/CA999999") + assert response.status_code == 404 + + +def test_get_allele_detail_is_public_no_privacy_gate( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """The decision: no privacy gate. An allele reachable only through a private score set is still + served to an anonymous caller — contrast ``GET /variants/{urn}``, which 404s the same variant.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + assert response.json()["digest"] == CDNA + + +def test_get_allele_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + """``as_of`` is echoed and reconstructs membership: before the record was live the class collapses + to the focus (the allele row itself is content-addressed, so it still resolves).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert response.status_code == 200 + assert response.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert set(response.json()["alleles"]) == {CDNA} diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py new file mode 100644 index 000000000..7ac06e709 --- /dev/null +++ b/tests/routers/test_clingen_allele.py @@ -0,0 +1,265 @@ +# ruff: noqa: E402 +"""Router tests for the ClinGen-allele measurements endpoint +(``GET /clingen-alleles/{caid}/measurements``). + +Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` +content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded +measurements are opt-in, a private score set's measurement never leaks, and the response body's +order reflects the ranked sort (current-then-superseded, direct-then-related, newest-published, urn). +""" + +from datetime import date + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + + +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", vrs_digest="cdna-digest"): + """Give a variant a live coding-measured mapping record whose authoritative allele carries the + ClinGen id — the minimal shape for a direct measurement on that CA's page. ``vrs_digest`` is + content-unique (DB constraint), so pass a distinct one when seeding more than one per test.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest=vrs_digest, level="cdna", clingen_allele_id=clingen_allele_id) + session.add(measured) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) + session.commit() + + +def test_measurements_serialize(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + AlleleMeasurement.model_validate(body[0]) + assert body[0]["variantUrn"] == urn + assert body[0]["relationship"] == "direct" + assert body[0]["assayLevel"] == "cdna" + assert body[0]["assayLevelHgvs"] == "NM_000546.6:c.1216G>A" + assert body[0]["submittedHgvs"] == "c.1A>T" + assert body[0]["scoreSetUrn"] == score_set["urn"] + assert body[0]["isCurrent"] is True + assert "supersededByScoreSet" not in body[0] # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_unknown_clingen_id_returns_empty_list(client, setup_router_db): + response = client.get("/api/v1/clingen-alleles/CA000/measurements") + assert response.status_code == 200 + assert response.json() == [] + + +def test_superseded_measurement_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1") + + default = client.get("/api/v1/clingen-alleles/CA123/measurements") + assert default.status_code == 200 + assert default.json() == [] + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + assert opted.status_code == 200 + body = opted.json() + assert len(body) == 1 + assert body[0]["isCurrent"] is False + assert body[0]["supersededByScoreSet"] == newer["urn"] + + +def test_nucleotide_siblings_shown(client, session, data_provider, data_files, setup_router_db): + """A different DNA variant encoding the same protein consequence surfaces on a CA page as a + ``nucleotide_encoding``, reached through the shared protein consequence.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + # Two coding measurements encoding the same protein consequence (PA9), each carrying its own CA. + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert response.status_code == 200 + by_urn = {m["variantUrn"]: m for m in response.json()} + assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} + assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" + assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" + + +def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): + """The list is ordered, and direct measurements precede related ones in the body: a CA query returns + the directly-assayed change first, then the sibling nucleotide encoding.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["relationship"]) for m in body] == [ + (f"{score_set['urn']}#1", "direct"), + (f"{score_set['urn']}#2", "nucleotide_encoding"), + ] + + +def test_current_measurements_sort_before_superseded(client, session, data_provider, data_files, setup_router_db): + """With superseded measurements opted in, current ones sort ahead of superseded ones in the body.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1", vrs_digest="cdna-older") + _seed_cdna_measurement(session, f"{newer['urn']}#1", vrs_digest="cdna-newer") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["isCurrent"]) for m in body] == [ + (f"{newer['urn']}#1", True), + (f"{older['urn']}#1", False), + ] + + +def test_measurements_sort_newest_published_first(client, session, data_provider, data_files, setup_router_db): + """Between two otherwise-equal current, unclassified measurements the newest-published sorts first — + published date outranks the urn tiebreak (here the newer set has the later urn, yet still leads).""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + first_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == first["urn"])) + second_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == second["urn"])) + first_db.published_date = date(2024, 1, 1) + second_db.published_date = date(2024, 6, 1) # later-published, and later urn + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + assert [m["variantUrn"] for m in response.json()] == [f"{second['urn']}#1", f"{first['urn']}#1"] + + +def test_measurements_tiebreak_on_urn(client, session, data_provider, data_files, setup_router_db): + """With every other sort key equal (both current, direct, unclassified, same published date) the urn is + the final, stable tiebreak — ascending.""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for urn in (first["urn"], second["urn"]): + score_set = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == urn)) + score_set.published_date = date(2024, 1, 1) + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + returned = [m["variantUrn"] for m in response.json()] + assert returned == sorted([f"{first['urn']}#1", f"{second['urn']}#1"]) + + +def test_anonymous_cannot_see_private_measurement( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_cdna_measurement(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + # A content-addressed ClinGen id is public, but a private score set's measurement is filtered out. + assert response.status_code == 200 + assert response.json() == [] diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py new file mode 100644 index 000000000..e118ba192 --- /dev/null +++ b/tests/routers/test_mapped_variant.py @@ -0,0 +1,53 @@ +# ruff: noqa: E402 +"""Router tests for the retired ``/mapped-variants`` surface. + +The router itself is gone (see ``tests/routers/test_variant_annotation.py``); what remains at this +prefix is a set of 301 redirects onto the routes' new home under ``/variants``, kept so external +callers hitting the old public paths get pointed at the replacement instead of a bare 404. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote, quote_plus + + +TEST_URN = "urn:mavedb:00000001-a-1#1" + + +def test_redirect_mapped_variant(client): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}" + + +@pytest.mark.parametrize( + "suffix", + ["va/study-result", "va/functional-statement", "va/pathogenicity-statement"], +) +def test_redirect_mapped_variant_va_routes(client, suffix): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}/{suffix}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}/{suffix}" + + +def test_redirect_mapped_variants_by_identifier(client): + identifier = "ga4gh:VA.0123456789abcdefghijklmnopqrstuv" + response = client.get( + f"/api/v1/mapped-variants/vrs/{identifier}?only_current=false", + follow_redirects=False, + ) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/vrs/{quote(identifier, safe='')}?only_current=false" + + +def test_redirect_mapped_variants_by_identifier_rejects_malformed_identifier(client): + response = client.get("/api/v1/mapped-variants/vrs/not-a-valid-identifier", follow_redirects=False) + + assert response.status_code == 422 diff --git a/tests/routers/test_mapped_variants.py b/tests/routers/test_mapped_variants.py deleted file mode 100644 index 51f45452f..000000000 --- a/tests/routers/test_mapped_variants.py +++ /dev/null @@ -1,620 +0,0 @@ -# ruff: noqa: E402 - -import json - -import pytest - -from tests.helpers.util.user import change_ownership - -arq = pytest.importorskip("arq") -cdot = pytest.importorskip("cdot") -fastapi = pytest.importorskip("fastapi") - -from urllib.parse import quote_plus - -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import select -from sqlalchemy.orm.session import make_transient - -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel -from mavedb.models.variant import Variant -from mavedb.view_models.mapped_variant import SavedMappedVariant -from tests.helpers.constants import ( - TEST_BIORXIV_IDENTIFIER, - TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, - TEST_PUBMED_IDENTIFIER, -) -from tests.helpers.util.common import deepcamelize -from tests.helpers.util.experiment import create_experiment -from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration -from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - create_seq_score_set_with_variants, -) - - -def test_show_mapped_variant(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["id"] == 1 - - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_when_multiple_exist(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_when_none_exists(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_show_mapped_variant_study_result(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." - - ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_study_result_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_study_result_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_cannot_show_mapped_variant_study_result_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No study result exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_functional_impact_statement( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." - - Statement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_functional_impact_statement_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_insufficient_functional_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - # insufficient evidence = no (primary) calibrations - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1. Variant does not have sufficient evidence to evaluate its functional impact" - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_clinical_evidence_line( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 200 - assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] - - VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_clinical_evidence_line_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_insufficient_pathogenicity_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1; Variant does not have sufficient evidence to evaluate its pathogenicity" - in response_data["detail"] - ) - - -def test_show_mapped_variants_by_ga4gh_identifier(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_when_none_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - fake_mapped_variant_id = mapped_variant.pre_mapped["id"] - fake_mapped_variant_id = fake_mapped_variant_id[:-3] + "aaa" # Modify the ID to ensure it doesn't exist - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(fake_mapped_variant_id)}") - assert response.status_code == 404 - - -def test_show_mapped_variants_by_ga4gh_identifier_with_non_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - # Set the mapped variant to non-current - mapped_variant.current = False - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=false" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_show_mapped_variants_by_ga4gh_identifier_with_only_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - mapped_variant2 = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#2')) - assert mapped_variant is not None - assert mapped_variant2 is not None - - # Set the mapped variant to non-current - mapped_variant2.current = False - mapped_variant2.pre_mapped = mapped_variant.pre_mapped # Ensure both pre mapped blobs match besides current status - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=true" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - 1 - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_without_score_set_permissions( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - change_ownership(session, score_set["urn"], ScoreSetDbModel) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - assert response.status_code == 404 diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 26a45bb32..27ecd6543 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4,9 +4,10 @@ import json import re from copy import deepcopy -from datetime import date +from datetime import date, datetime, timezone from io import StringIO -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import jsonschema import pytest @@ -21,15 +22,17 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.enums.target_category import TargetCategory from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline -from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel -from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel +from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.lean_variant import LeanVariant +from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel from mavedb.routers.score_sets import _annotation_stream_record from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate @@ -64,10 +67,12 @@ TEST_SAVED_GNOMAD_VARIANT, TEST_SAVED_TAXONOMY, TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider -from tests.helpers.mocks.factories import create_mock_mapped_variant +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.common import ( create_failing_side_effect, deepcamelize, @@ -85,15 +90,13 @@ create_seq_score_set, create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, - link_clinical_controls_to_mapped_variants, - link_clinvar_control_to_mapped_variant, - link_gnomad_variants_to_mapped_variants, + link_clinical_controls_to_alleles, publish_score_set, + seed_annotation_substrate, + seed_csv_substrate, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - clear_first_mapped_variant_post_mapped, - create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -1541,7 +1544,7 @@ def test_upload_score_set_variant_data_deletes_s3_files_when_pipeline_creation_f with ( open(scores_csv_path, "rb") as scores_file, - patch("mavedb.routers.score_sets.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), + patch("mavedb.lib.workflow.kickoff.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), patch.object(mock_s3_client, "upload_fileobj", return_value=None), ): response = client.post( @@ -2052,7 +2055,10 @@ def test_multiple_score_set_meta_analysis_single_experiment( published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2092,7 +2098,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2134,7 +2143,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiments( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2190,7 +2202,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco published_score_set_1_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1_1['urn']}")).json() assert meta_score_set_1["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1_1["urn"], published_score_set_1_2["urn"]] + [ + published_score_set_1_1["urn"], + published_score_set_1_2["urn"], + ] ) assert published_score_set_1_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_1["urn"]] @@ -2207,7 +2222,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco ) published_score_set_2_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_2_1['urn']}")).json() assert meta_score_set_2["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_2_1["urn"], published_score_set_2_2["urn"]] + [ + published_score_set_2_1["urn"], + published_score_set_2_2["urn"], + ] ) assert published_score_set_2_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_2["urn"]] @@ -2329,7 +2347,10 @@ def test_multiple_score_set_meta_analysis_single_experiment_with_different_creat published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2371,7 +2392,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_with_differen published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -3426,7 +3450,17 @@ def test_download_variants_data_file( score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3540,7 +3574,12 @@ def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published_score_set = publish_score_set(client, score_set["urn"]) @@ -3718,7 +3757,16 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r download_scores_and_counts_csv = download_scores_and_counts_csv_response.text reader = csv.DictReader(StringIO(download_scores_and_counts_csv)) assert sorted(reader.fieldnames) == sorted( - ["accession", "hgvs_nt", "hgvs_pro", "scores.score", "scores.s_0", "scores.s_1", "counts.c_0", "counts.c_1"] + [ + "accession", + "hgvs_nt", + "hgvs_pro", + "scores.score", + "scores.s_0", + "scores.s_1", + "counts.c_0", + "counts.c_1", + ] ) @@ -3740,7 +3788,17 @@ def test_download_scores_counts_and_post_mapped_variants_file( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3778,8 +3836,14 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + # Seed mapped alleles with a VEP consequence on the authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3802,13 +3866,15 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants then set ClinGen allele id for first mapped variant - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles, with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3826,11 +3892,31 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie def test_download_gnomad_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) - # Link a gnomAD variant to the first mapped variant (version may not match export filter) - score_set = create_seq_score_set_with_mapped_variants( - client, session, data_provider, experiment["urn"], data_files / "scores.csv" + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # The CSV gnomAD column reflects the served release (v4.1); seed one and link it to the first variant's + # authoritative allele. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v4.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + annotate="first", ) - link_gnomad_variants_to_mapped_variants(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3842,21 +3928,47 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames + rows = list(reader) + assert rows[0]["gnomad.gnomad_af"] == str(3 / 1613510) + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows[1:]) -def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( +def test_download_gnomad_file_serves_variants_linked_to_another_release( session, data_provider, client, setup_router_db, data_files ): - """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + """A variant whose live gnomAD link is to an older release is served, labeled with that release. - The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + Two properties at once. The row must survive regardless of gnomAD linkage — the join's ON clause is + what preserves it, and the same predicate in a WHERE silently drops the variant. And the frequency + itself is read off the live link rather than filtered to the release the deployment currently serves: + ingestion only visits alleles a release covers, so alleles sitting at an older release are the steady + state, and filtering them out made their frequencies permanently NA. """ experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - # The seeded gnomAD variant's version deliberately differs from the configured export version. - link_gnomad_variants_to_mapped_variants(session, score_set) + # Deliberately not the release the deployment serves (v4.1): this is an allele a newer release did + # not cover, so its earlier link is still the live one. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v2.1.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3867,7 +3979,8 @@ def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( rows = list(csv.DictReader(StringIO(response.text))) assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" - assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + assert all(row["gnomad.gnomad_af"] == str(3 / 1613510) for row in rows) + assert all(row["gnomad.gnomad_version"] == "v2.1.1" for row in rows) def test_download_clingen_and_vep_file_in_variant_data_path( @@ -3878,13 +3991,16 @@ def test_download_clingen_and_vep_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a VEP consequence (all variants) and a ClinGen id (first variant only). + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3910,13 +4026,15 @@ def test_download_clingen_and_scores_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3948,10 +4066,18 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, # The ClinVar control seeded in setup_router_db has db_version="11_2024", mapping to namespace clinvar.2024_11. clinvar_namespace = "clinvar.2024_11" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + # ClinVar control id=1 (db_version 11_2024) links to only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3982,10 +4108,17 @@ def test_download_clinvar_namespace_with_no_matching_version( # clinvar.2023_01 does not match the seeded control (11_2024), so all rows should be NA. clinvar_namespace = "clinvar.2023_01" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4012,10 +4145,17 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( matching_ns = "clinvar.2024_11" # matches db_version="11_2024" seeded in setup_router_db non_matching_ns = "clinvar.2023_01" # no controls with this version experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4043,6 +4183,91 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( assert all(row[f"{non_matching_ns}.clinical_review_status"] == "NA" for row in rows) +def test_csv_namespaces_as_of_matches_what_the_as_of_download_can_fill( + session, data_provider, client, setup_router_db, data_files +): + """Discovery and download must answer for the same instant. + + Pinned to "now", the picker offers mapping namespaces an `as_of` download returns entirely NA, and + omits ones it could have filled. The ClinVar case is sharpest: the live release set decides which + `clinvar.YYYY_MM` headers exist at all. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + url = f"/api/v1/score-sets/{published['urn']}/csv-namespaces" + + current = client.get(url) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert "vep" in {entry["namespace"] for entry in current.json()} + + # Before the mapping was live: the mapping-derived namespaces are not offered, because the + # `as_of` download at that instant could not fill them. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(url, params={"as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert "vep" not in {entry["namespace"] for entry in before.json()} + + +def test_download_variant_data_as_of_reconstructs_annotation_layer( + session, data_provider, client, setup_router_db, data_files +): + """`as_of` time-travels the CSV annotation layer: an instant before the annotation was live yields NA, + while the current view yields the value. The resolved instant is echoed in X-As-Of.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # Seed the whole substrate live as of 2020 — live "now" but not at an as_of in 2000. + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + base_url = f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + + # Current: the annotation is live, and X-As-Of echoes "current". + current = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + current_rows = list(csv.DictReader(StringIO(current.text))) + assert any(row["vep.vep_functional_consequence"] == "missense_variant" for row in current_rows) + + # An instant before the annotation existed: nothing live -> every VEP value is NA, and X-As-Of echoes it. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert datetime.fromisoformat(before.headers["X-As-Of"]) == past + before_rows = list(csv.DictReader(StringIO(before.text))) + assert before_rows # variants (and their immutable scores) are still present + assert all(row["vep.vep_functional_consequence"] == "NA" for row in before_rows) + + def test_invalid_clinvar_namespace_returns_422(client, setup_router_db, data_files): """A clinvar namespace with an out-of-range month (13) is rejected with 422.""" experiment = create_experiment(client) @@ -4071,7 +4296,7 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") assert response.status_code == 200 @@ -4079,15 +4304,66 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ response_data = response.json() assert len(response_data) == 2 for control in response_data: - mapped_variants = control.pop("mappedVariants") - assert len(mapped_variants) == 1 + clinvar_links = control.pop("clinvarLinks") + assert len(clinvar_links) == 1 + # Each link carries the digest of the allele the control annotates — the D78 precedence signal. + assert clinvar_links[0]["alleleDigest"] in ("clinical-control-allele-0", "clinical-control-allele-1") assert all( control[k] in (TEST_SAVED_CLINVAR_CONTROL[k], TEST_SAVED_GENERIC_CLINICAL_CONTROL[k]) for k in TEST_SAVED_CLINVAR_CONTROL.keys() - if k != "mappedVariants" + if k != "clinvarLinks" ) +def test_clinical_controls_tag_each_link_with_the_annotating_allele_digest( + client, setup_router_db, session, data_provider, data_files +): + """A protein-change variant whose DNA siblings carry conflicting ClinVar calls surfaces *both* + controls against the *same* variant URN — distinguishable only by the digest of the allele each + one annotates. That digest is D78's precedence signal: the control on the variant's authoritative + (assayed-level) allele is the direct call; the control on a projection sibling is the fallback. + Without it the two collapse to one variant and precedence is unimplementable.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = session.scalars( + select(VariantDbModel.urn) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).first() + + # One variant, two alleles in its equivalence class: the assayed (authoritative) allele carries the + # ClinVar control (id 1); a projection sibling carries the generic control (id 2). + seed_mapping_record( + session, + variant_urn, + assay_level="protein", + alleles=[ + AlleleSpec(digest="assayed-level-allele", level="protein", is_authoritative=True, clinvar_control_ids=[1]), + AlleleSpec( + digest="projection-sibling-allele", level="cdna", is_authoritative=False, clinvar_control_ids=[2] + ), + ], + ) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") + assert response.status_code == 200 + + controls = {c["dbIdentifier"]: c for c in response.json()} + assert len(controls) == 2 + + assayed = controls[TEST_SAVED_CLINVAR_CONTROL["dbIdentifier"]] # control 1, on the authoritative allele + sibling = controls[TEST_SAVED_GENERIC_CLINICAL_CONTROL["dbIdentifier"]] # control 2, on the sibling + + # Both controls point at the same variant, so only the annotating allele's digest tells them apart. + assert [link["variantUrn"] for link in assayed["clinvarLinks"]] == [variant_urn] + assert [link["variantUrn"] for link in sibling["clinvarLinks"]] == [variant_urn] + assert assayed["clinvarLinks"][0]["alleleDigest"] == "assayed-level-allele" + assert sibling["clinvarLinks"][0]["alleleDigest"] == "projection-sibling-allele" + + @pytest.mark.parametrize("clinical_control", [TEST_SAVED_CLINVAR_CONTROL, TEST_SAVED_GENERIC_CLINICAL_CONTROL]) @pytest.mark.parametrize( "parameters", [[("db", "dbName")], [("version", "dbVersion")], [("db", "dbName"), ("version", "dbVersion")]] @@ -4099,7 +4375,7 @@ def test_can_fetch_current_clinical_controls_for_score_set_with_parameters( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) query_string = "?" for param, accessor in parameters: @@ -4124,7 +4400,7 @@ def test_cannot_fetch_clinical_controls_for_nonexistent_score_set( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/clinical-controls") @@ -4159,7 +4435,7 @@ def test_can_fetch_current_clinical_control_options_for_score_set( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 200 @@ -4184,23 +4460,176 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) - # Mark all mapped variants as non-current to simulate stale mapping data. - mapped_variants = session.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - for mv in mapped_variants: - mv.current = False + # The options query only surfaces live allele → ClinVar links; retiring them (as a re-map would, + # by stamping valid_to) must leave no current controls and 404. Test-isolated DB, so these are + # the only links present. + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.valid_to.is_(None))).all() + for link in links: + link.valid_to = datetime.now(timezone.utc) session.commit() response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 404 +######################################################################################################################## +# Downloading a score set's variant details (VRS + Cat-VRS + annotations) +######################################################################################################################## + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_get_score_set_variant_details_withholds_a_private_calibration( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + """Reading a score set does not entitle a caller to a private calibration's classifications. + + The whole-set export resolved calibration visibility by trusting the fetch helper to have narrowed + ``score_calibrations`` in place. That narrowing moved into the response builder, which this route does + not go through, so every calibration on the score set became visible to anyone who could read it -- + the single-variant twin filters correctly, and the two serve the same envelope. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + + # A community calibration on someone else's score set: private, and readable only by its creator. + calibration_item = session.query(ScoreCalibrationDbModel).filter_by(urn=calibration["urn"]).one() + calibration_item.investigator_provided = False + session.commit() + change_ownership(session, calibration["urn"], ScoreCalibrationDbModel) + + owner_records = parse_ndjson_response(client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details")) + assert owner_records, "the score set's own variants must still stream" + assert not any( + classification.get("calibrationId") == calibration_item.id + for record in owner_records + for classification in (record.get("classifications") or []) + ), "a calibration the caller cannot read must not contribute classifications" + + +def test_get_score_set_variant_details_returns_ndjson(client, session, data_provider, data_files, setup_router_db): + """The whole-set variant-detail export streams NDJSON — one VariantDetail per mapped variant, each + carrying the flat preMapped/postMapped VRS pair and the spec-pure Cat-VRS.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urns = { + variant.urn + for variant in seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + } + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/x-ndjson") + assert response.headers["X-As-Of"] == "current" + assert response.headers["X-Total-Count"] == str(len(variant_urns)) + assert response.headers["X-Stream-Type"] == "variant-detail" + + records = parse_ndjson_response(response) + assert len(records) == len(variant_urns) + assert {record["urn"] for record in records} == variant_urns + for record in records: + # The flat VRS pair for VRS consumers, and the spec-pure Cat-VRS for the full picture. + assert record["preMapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert record["postMapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert record.get("molecularRepresentation") is not None + + +def test_get_score_set_variant_details_omits_unmapped_variants( + client, session, data_provider, data_files, setup_router_db +): + """Only mapped variants carry VRS/detail, so an un-mapped variant is omitted — the record count is + the mapped subset, not the whole score set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variants = seed_annotation_substrate(session, score_set, skip_first=True) + unmapped_urn = variants[0].urn # read before the request expires the instances + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + + records = parse_ndjson_response(response) + assert len(records) == score_set["numVariants"] - 1 + assert unmapped_urn not in {record["urn"] for record in records} + + +def test_get_score_set_variant_details_for_unmapped_returns_empty( + client, session, data_provider, data_files, setup_router_db +): + """A score set with no live mapping records has no annotatable variants — an empty NDJSON stream with + X-Total-Count 0, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # No new-substrate mapping records → no annotatable variants. + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variant-details") + assert response.status_code == 404 + + +def test_score_set_mapped_variants_is_permanently_removed(client): + """The old JSON-array ``mapped-variants`` route is gone in favor of the streaming NDJSON + ``variant-details`` route; it returns 410 rather than redirecting since the two are not + wire-compatible.""" + urn = "urn:mavedb:00000001-a-1" + response = client.get(f"/api/v1/score-sets/{urn}/mapped-variants") + + assert response.status_code == 410 + assert response.json()["detail"] == ( + f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET /score-sets/{urn}/variant-details instead." + ) + + +def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate + as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — + ``as_of`` is a filter, so an empty match is an empty collection, never a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + url = f"/api/v1/score-sets/{score_set['urn']}/variant-details" + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] + + ######################################################################################################################## # Fetching annotated variants for a score set ######################################################################################################################## @@ -4219,9 +4648,14 @@ def test_cannot_get_annotated_variants_for_nonexistent_score_set(client, setup_r @pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) -def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( +def test_get_annotated_variants_for_score_set_with_no_mapped_variants_streams_empty( client, session, data_provider, data_files, setup_router_db, annotation_type ): + """A score set that exists but has no annotatable variants is an empty collection, not a 404. + + The route resolves (the URN names a real, readable score set), so it returns 200 with an empty NDJSON + body and ``X-Total-Count: 0``. 404 is reserved for an unresolvable URN or a permission failure. + """ experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -4246,13 +4680,47 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( assert "hgvs_splice" not in columns response = client.get(f"/api/v1/score-sets/{publish_score_set['urn']}/annotated-variants/{annotation_type}") - response_data = response.json() - assert response.status_code == 404 - assert ( - f"No mapped variants associated with score set URN {publish_score_set['urn']} were found" - in response_data["detail"] + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +@pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) +def test_annotated_variants_honor_as_of(client, session, data_provider, data_files, setup_router_db, annotation_type): + """The streaming annotation routes time-travel their annotatable set via ``as_of``. + + ``as_of`` is threaded into ``get_annotatable_variants`` (which allele links are live at that instant), + not only into the per-variant context. So a far-future instant sees the freshly-seeded substrate as + live (the full set streams, and the route echoes the instant on ``X-As-Of``); a far-past instant sees + no live mapping data (the annotatable set is empty). Either way the route returns 200 — ``as_of`` is a + filter, so an empty match is an empty collection, never a 404. This pins the wiring so the set + enumeration and the per-variant annotation cannot drift to different instants. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/{annotation_type}" + + # Far future: the seeded mapping records are live, so the whole set streams and X-As-Of echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + # Far past: nothing was live yet, so the annotatable set is empty — an empty 200 stream, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + + assert past_response.status_code == 200 + assert datetime.fromisoformat(past_response.headers["X-As-Of"]) == past + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] # Tests that annotated variants of the correct type are returned when appropriate. The contents of these @@ -4283,6 +4751,7 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) # The contents of the annotated variants objects should be tested in more detail elsewhere. response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") @@ -4314,6 +4783,10 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_thre data_files / "scores.csv", ) + # Variants are mapped on the allele substrate, so they are annotatable; the null annotation comes from + # the missing calibration/thresholds, not from missing mapping. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4337,6 +4810,9 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_cali data_files / "scores.csv", ) + # Variants are mapped on the allele substrate (so annotatable); the null comes from the absent calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4373,21 +4849,21 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set_when_some_vari client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") + assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") @pytest.mark.parametrize( @@ -4414,6 +4890,7 @@ def test_get_annotated_functional_impact_statement_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4447,6 +4924,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_calib }, ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent promoted calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4470,6 +4950,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_thres data_files / "scores.csv", ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent calibration/ranges. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4506,21 +4989,20 @@ def test_get_annotated_functional_impact_statement_for_score_set_when_some_varia client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "Statement" + assert annotated_variant.get("type") == "Statement" @pytest.mark.parametrize( @@ -4539,6 +5021,7 @@ def test_get_annotated_functional_study_result_for_score_set( experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4571,6 +5054,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4603,6 +5087,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_ranges_not_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4626,6 +5111,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4659,21 +5145,20 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian }, ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( @@ -4681,9 +5166,10 @@ def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( ): """One variant that cannot be annotated must not cost the consumer the rest of the download.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) failing_annotation = create_failing_side_effect( # Representative of lib/annotation/util.py, which raises this on an unrecognized VRS Allele state. @@ -4715,11 +5201,10 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( ): """Every line is a variant record, so a body shorter than X-Total-Count is a truncated one.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - unmapped_variant = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) - assert unmapped_variant is not None + unmapped_variant_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn failing_annotation = create_failing_side_effect( ValueError("Unsupported VRS state type"), variant_study_result, fail_on_call=2 @@ -4728,19 +5213,19 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + # X-Total-Count counts annotatable variants; the unmapped one is not among them. total_count = int(response.headers["X-Total-Count"]) - assert total_count == score_set["numVariants"] + assert total_count == score_set["numVariants"] - 1 response_data = parse_ndjson_response(response) + # The header and the body must agree — a shorter body is a truncated one. assert len(response_data) == total_count assert all("variant_urn" in record for record in response_data) + assert unmapped_variant_urn not in {record["variant_urn"] for record in response_data} - # A variant with no mapping data is an expected absence, distinguishable from the failure. + # The failure is reported in-band rather than by ending the stream early. errored = [record for record in response_data if "error" in record] - unannotated = [record for record in response_data if record["annotation"] is None and "error" not in record] assert len(errored) == 1 - assert len(unannotated) == 1 - assert unannotated[0]["variant_urn"] == unmapped_variant.urn ######################################################################################################################## @@ -4751,6 +5236,10 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( ######################################################################################################################## +#: Returned by the stubbed context factory; the stub annotation functions never read it. +_STUB_CONTEXT = object() + + class _StubAnnotation: def model_dump(self, **kwargs): return {"type": "Stub"} @@ -4764,37 +5253,58 @@ def model_dump(self, **kwargs): def _annotation_raising(exception): """An annotation function that fails.""" - def annotate(_mapped_variant): + def annotate(_context): raise exception return annotate @pytest.fixture -def mock_mapped_variant_for_stream(): - return create_mock_mapped_variant(clingen_allele_id="CA123456") +def stream_variant(): + """Stands in for the annotatable variant the stream iterates; only its URN is read here.""" + return SimpleNamespace(urn="tmp:test-urn#1") + +def _record_for(variant, annotation_function, context=_STUB_CONTEXT): + """Drive one stream record with the context factory stubbed out. + + ``context`` is what the factory returns; pass None for the variant whose mapping substrate yields + nothing to annotate. + """ + with patch("mavedb.routers.score_sets.variant_annotation_context", return_value=context): + return _annotation_stream_record(MagicMock(), variant, annotation_function) -def test_annotation_stream_record_serializes_a_successful_annotation(mock_mapped_variant_for_stream): - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _StubAnnotation()) + +def test_annotation_stream_record_serializes_a_successful_annotation(stream_variant): + record, outcome = _record_for(stream_variant, lambda ctx: _StubAnnotation()) assert outcome == "annotated" - assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": {"type": "Stub"}} + assert record == {"variant_urn": stream_variant.urn, "annotation": {"type": "Stub"}} -def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(mock_mapped_variant_for_stream): +def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(stream_variant): """A variant the annotation layer declines to annotate is an expected outcome, not a failure.""" - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: None) + record, outcome = _record_for(stream_variant, lambda ctx: None) + + assert outcome == "unannotated" + assert record == {"variant_urn": stream_variant.urn, "annotation": None} + + +def test_annotation_stream_record_treats_an_absent_context_as_unannotated(stream_variant): + """No live mapping record at ``as_of`` means nothing to annotate, so the builder is never called.""" + record, outcome = _record_for( + stream_variant, _annotation_raising(AssertionError("must not be called")), context=None + ) assert outcome == "unannotated" - assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": None} + assert record == {"variant_urn": stream_variant.urn, "annotation": None} -def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(mock_mapped_variant_for_stream): +def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(stream_variant): # Preserved deliberately: a missing mapping is an expected absence, and reporting it as an error would # tell consumers a variant failed when nothing went wrong. - record, outcome = _annotation_stream_record( - mock_mapped_variant_for_stream, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) + record, outcome = _record_for( + stream_variant, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) ) assert outcome == "unannotated" @@ -4813,22 +5323,22 @@ def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(moc IndexError("list index out of range"), ], ) -def test_annotation_stream_record_reports_any_other_failure_as_an_error(mock_mapped_variant_for_stream, exception): - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, _annotation_raising(exception)) +def test_annotation_stream_record_reports_any_other_failure_as_an_error(stream_variant, exception): + record, outcome = _record_for(stream_variant, _annotation_raising(exception)) assert outcome == "errored" - assert record["variant_urn"] == mock_mapped_variant_for_stream.variant.urn + assert record["variant_urn"] == stream_variant.urn assert record["annotation"] is None assert record["error"] == {"type": type(exception).__name__, "detail": str(exception)} -def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mock_mapped_variant_for_stream): +def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(stream_variant): """An emitted object that no longer dumps is the shape-dependent failure this stream must survive. Commit 5c155f4d fixed exactly this: a required field combined with `exclude_none` produced an object that built successfully and then failed on the way out. """ - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _UndumpableAnnotation()) + record, outcome = _record_for(stream_variant, lambda ctx: _UndumpableAnnotation()) assert outcome == "errored" assert record["error"] == {"type": "ValueError", "detail": "Extension.value is required"} @@ -4841,19 +5351,32 @@ def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mo def test_can_fetch_current_gnomad_variants_for_score_set(client, setup_router_db, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + # gnomAD variant id=1 (seeded by setup_router_db) links to the first variant's authoritative allele. + variants = seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) + expected_urn = variants[0].urn + expected_digest = f"csv-auth-{variants[0].id}" response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants") assert response.status_code == 200 + assert response.headers["x-as-of"] == "current" response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 + assert variant_links[0]["variantUrn"] == expected_urn + assert variant_links[0]["alleleDigest"] == expected_digest gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4862,10 +5385,17 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version={TEST_GNOMAD_DATA_VERSION}") assert response.status_code == 200 @@ -4873,8 +5403,8 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4883,10 +5413,17 @@ def test_cannot_fetch_current_gnomad_variants_for_score_set_with_nonexistent_ver client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version=nonexistent_version") assert response.status_code == 404 @@ -4903,10 +5440,17 @@ def test_cannot_fetch_gnomad_variants_for_nonexistent_score_set( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/gnomad-variants") @@ -4919,7 +5463,7 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) @@ -4933,6 +5477,171 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( ) +def _seed_lean_mapping(session, variant_urn): + """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose + authoritative allele carries a digest + ClinGen id + a live VEP consequence, plus its canonical + genomic projection sibling (shared ``projection_group``) and the protein apex — the full triple.""" + seed_mapping_record( + session, + variant_urn, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + alleles=[ + AlleleSpec( + digest="cdna-digest", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-digest", + level="genomic", + hgvs_g="NC_000017.11:g.7676154C>T", + projection_group=0, + ), + AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), + ], + ) + + +def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_lean_mapping(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + records = response.json() + # All three variants are returned, ordered by variant number. + assert [r["variantUrn"] for r in records] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + for record in records: + LeanVariant.model_validate_json(json.dumps(record)) + + mapped = records[0] + assert mapped["score"] == 0.3 + assert mapped["consequence"] == "missense_variant" + assert mapped["clingenAlleleId"] == "CA123" + assert mapped["assayLevelDigest"] == "cdna-digest" + # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. + assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} + assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} + # The mapped triple (reference frame): the measured slot named by assayLevel, its projection_group + # genomic sibling, and the protein apex. mapped.cdna is the search key even here (measured at cdna). + assert mapped["assayLevel"] == "cdna" + assert mapped["mapped"]["cdna"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["mapped"]["genomic"] == { + "hgvs": "NC_000017.11:g.7676154C>T", + "position": 7676154, + "ref": "C", + "alt": "T", + } + assert mapped["mapped"]["protein"] == { + "hgvs": "NP_000537.3:p.Ala406Thr", + "position": 406, + "ref": "Ala", + "alt": "Thr", + } + + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none), + # and the mapped triple serializes empty (no slots) since none is populated. + unmapped = records[1] + assert unmapped["score"] == 1.0 + assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevel"): + assert omitted not in unmapped + assert unmapped["mapped"] == {} + + +def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variants") + assert response.status_code == 404 + + +def test_get_lean_variants_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Default is current — the resolved content-time is echoed for the client. + current = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + + # An explicit as_of is accepted and echoed back; before mapping exists everything is unmapped. + historical = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants", params={"as_of": "2020-01-01T00:00:00Z"}) + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert all("assayLevelDigest" not in record for record in historical.json()) + + +def test_get_lean_variants_other_user_cannot_read_private(client, session, data_provider, data_files, setup_router_db): + """The endpoint gates on READ: a non-owner cannot read another user's private (unpublished) set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + assert f"score set with URN '{score_set['urn']}' not found" in response.json()["detail"] + + +def test_get_lean_variants_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + + +def test_get_lean_variants_anonymous_can_read_published( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """A published score set is world-readable, so an anonymous caller gets the full lean set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + +def test_get_lean_variants_admin_can_read_private( + client, session, data_provider, data_files, setup_router_db, admin_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(admin_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + @pytest.mark.parametrize( "mock_publication_fetch", [ diff --git a/tests/routers/test_statistics.py b/tests/routers/test_statistics.py index 69be6ffba..21ae62676 100644 --- a/tests/routers/test_statistics.py +++ b/tests/routers/test_statistics.py @@ -21,7 +21,12 @@ TEST_PUBMED_IDENTIFIER, VALID_GENE, ) -from tests.helpers.util.score_set import publish_score_set, create_acc_score_set, create_seq_score_set +from tests.helpers.util.score_set import ( + publish_score_set, + create_acc_score_set, + create_seq_score_set, + seed_annotation_substrate, +) from tests.helpers.util.experiment import create_experiment from tests.helpers.util.variant import mock_worker_variant_insertion, create_mapped_variants_for_score_set @@ -65,7 +70,12 @@ def setup_seq_scoreset(setup_router_db, session, data_provider, client, data_fil unpublished_score_set = mock_worker_variant_insertion( client, session, data_provider, unpublished_score_set, data_files / "scores.csv" ) + # Fully-mapped state: target post_mapped_metadata + mapping_state (read by the target-gene + # statistics). MappedVariant rows are seeded here too but are no longer read by the MV. create_mapped_variants_for_score_set(session, unpublished_score_set["urn"], TEST_MINIMAL_MAPPED_VARIANT) + # The published_variants MV reads the MappingRecord/Allele substrate now (not MappedVariant), so + # give each variant a live mapping record for the mapped-variant statistics to count. + seed_annotation_substrate(session, unpublished_score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: publish_score_set(client, unpublished_score_set["urn"]) diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 48f3fbd99..a650e3d45 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -1,6 +1,15 @@ # ruff: noqa: E402 +"""Router tests for the assayed variant-detail endpoint (``GET /variants/{urn}``) and the +single-variant CSV export (``GET /variants/{urn}/csv``). + +Exercise the HTTP surface end to end: the two-tier envelope serializes and validates, the ``as_of`` +content-time is echoed in the ``X-As-Of`` header, a superseded variant self-describes rather than +reading as current, the READ gate holds for private score sets, and a private calibration is never +served over either surface. +""" import csv +from datetime import datetime, timezone from io import StringIO from unittest.mock import patch from urllib.parse import quote @@ -11,16 +20,224 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from sqlalchemy import select +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.variant_detail import VariantDetail from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.experiment import create_experiment from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - link_clinvar_control_to_mapped_variant, + create_seq_score_set_with_variants, + link_clinical_controls_to_alleles, publish_score_set, + seed_csv_substrate, ) +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this on the detail path.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed_mapping(session, variant_urn): + """Give a variant a live coding-measured mapping record whose authoritative allele carries a + digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared + projection_group), a protein apex, and a synonymous cousin (a nt encoder of the same consequence in + a different projection group) — enough to build Cat-VRS and exercise the projection + convergent axes.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele( + vrs_digest="cdna-digest", + level="cdna", + post_mapped=_post_mapped(), + clingen_allele_id="CA123", + hgvs_c="NM_000546.6:c.1216G>A", + ) + genomic = Allele( + vrs_digest="gen-digest", level="genomic", post_mapped=_post_mapped(), hgvs_g="NC_000017.11:g.7676154C>T" + ) + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" + ) + cousin = Allele( + vrs_digest="cousin-digest", level="cdna", post_mapped=_post_mapped(), hgvs_c="NM_000546.6:c.1218C>T" + ) + session.add_all([measured, genomic, protein, cousin]) + session.commit() + + session.add_all( + [ + # The measured cdna link and its genomic projection share a projection_group; the protein + # apex is in no pair (group None); the cousin sits in its own projection group. + MappingRecordAllele( + mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 + ), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 + ), + MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=cousin.id, is_authoritative=False, projection_group=1 + ), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_variant_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_mapping(session, urn) + + response = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}") + + assert response.status_code == 200 + body = response.json() + VariantDetail.model_validate(body) + + assert body["urn"] == urn + assert body["assayLevel"] == "cdna" + assert body["referenceHgvs"] == "NM_000546.6:c.1216G>A" + assert body["assayLevelDigest"] == "cdna-digest" + assert body["clingenAlleleId"] == "CA123" + assert body["mode"] == "projection" + # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). + assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar (every linked allele is a + # member, the cousin included), so the envelope is internally consistent. + assert len(body["molecularRepresentation"]["members"]) == len(body["alleles"]) + # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a + # translation of the defining coding allele (which itself has no relation to itself). + assert body["alleles"]["cdna-digest"]["level"] == "cdna" + assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" + assert ( + body["alleles"]["cdna-digest"]["relation"] is None + ) # measured allele: no relation to itself (null, not dropped) + # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the + # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). + assert body["alleles"]["cdna-digest"]["isFocus"] is True + assert body["alleles"]["cdna-digest"]["derivation"] is None # null, not dropped (exclude_none=False) + assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["isFocus"] is False + assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" + assert body["alleles"]["gen-digest"]["derivation"] == "projection" + assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" + assert body["alleles"]["prot-digest"]["level"] == "protein" + assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" + assert body["alleles"]["prot-digest"]["relation"] == "translation_of" + # The apex is a deterministic projection here but pairs with nothing (projectionOf is null). + assert body["alleles"]["prot-digest"]["derivation"] == "projection" + assert body["alleles"]["prot-digest"]["projectionOf"] is None + # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is + # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). + assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" + assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" + assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" + assert body["isCurrent"] is True + assert body["supersededByScoreSet"] is None # null when current (stable envelope, not dropped) + assert response.headers["X-As-Of"] == "current" + + +def test_get_variant_detail_unknown_urn_is_404(client, setup_router_db): + response = client.get("/api/v1/variants/urn:mavedb:00000000-a-1#1") + assert response.status_code == 404 + + +def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + + historical = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + # Before any mapping existed, the molecular layer is empty — the stable envelope carries it as null. + assert historical.json()["molecularRepresentation"] is None + + +def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): + """A variant on a superseded score set is still served, but flags its version standing.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Link newer as the version that replaces older (replaces_id / superseding relationship). + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + + response = client.get(f"/api/v1/variants/{older['urn']}%231") + + assert response.status_code == 200 + body = response.json() + assert body["isCurrent"] is False + assert body["supersededByScoreSet"] == newer["urn"] + + +def test_get_variant_detail_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/variants/{score_set['urn']}%231") + + assert response.status_code == 404 def _first_variant_urn(session, score_set_urn): @@ -33,11 +250,12 @@ def _csv_path(variant_urn): return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" -def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): +def _published_score_set_with_mappings(client, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_csv_substrate(session, score_set, assay_level="genomic", hgvs_g="NC_000010.11:g.1A>G") with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) return published @@ -45,7 +263,7 @@ def _published_score_set_with_mapped_variants(client, session, data_provider, da class TestGetVariantCsv: def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(_csv_path(variant_urn)) @@ -55,7 +273,7 @@ def test_returns_csv_attachment(self, session, data_provider, client, setup_rout assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(_csv_path(variant_urn)) @@ -67,7 +285,7 @@ def test_reports_the_requested_variant(self, session, data_provider, client, set assert rows[0]["relationship.match_type"] == "exact" def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") @@ -82,10 +300,10 @@ def test_clinvar_namespace_is_labeled_with_its_release( self, session, data_provider, client, setup_router_db, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + link_clinical_controls_to_alleles(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) variant_urn = _first_variant_urn(session, published["urn"]) @@ -104,7 +322,7 @@ def test_invalid_namespace_is_rejected( self, session, data_provider, client, setup_router_db, data_files, namespace ): """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") @@ -115,7 +333,7 @@ def test_invalid_namespace_is_rejected( @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") @@ -157,7 +375,7 @@ def test_private_score_set_is_not_readable_by_other_users( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) variant_urn = _first_variant_urn(session, score_set["urn"]) @@ -198,7 +416,7 @@ def _private_calibration(self, session, score_set_urn): def test_another_user_naming_the_urn_gets_no_interpretation( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) calibration_urn = self._private_calibration(session, published["urn"]) namespace = f"calibration.{calibration_urn}" @@ -215,7 +433,7 @@ def test_another_user_naming_the_urn_gets_no_interpretation( def test_another_user_is_not_offered_it_by_discovery( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) calibration_urn = self._private_calibration(session, published["urn"]) with DependencyOverrider(extra_user_app_overrides): @@ -225,13 +443,77 @@ def test_another_user_is_not_offered_it_by_discovery( assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] +class TestVariantCsvAsOf: + """`as_of` time-travels the single-variant CSV, as it does the score-set CSV. + + The library call has always supported it; the route did not expose it, so the whole reconstruction + path was unreachable over HTTP and the response did not even self-describe its content-time. + """ + + def _seeded(self, client, session, data_provider, data_files, seeded_at): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000010.11:g.1A>G", + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + return publish_score_set(client, score_set["urn"]) + + def test_csv_echoes_and_reconstructs_at_the_requested_instant( + self, session, data_provider, client, setup_router_db, data_files + ): + published = self._seeded(client, session, data_provider, data_files, datetime(2020, 1, 1, tzinfo=timezone.utc)) + variant_urn = _first_variant_urn(session, published["urn"]) + + current = client.get(_csv_path(variant_urn), params={"namespaces": "vep"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert any( + row["vep.vep_functional_consequence"] == "missense_variant" + for row in csv.DictReader(StringIO(current.text)) + ) + + # Before the mapping was live, the same columns reconstruct as NA. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(_csv_path(variant_urn), params={"namespaces": "vep", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert all(row["vep.vep_functional_consequence"] == "NA" for row in csv.DictReader(StringIO(before.text))) + + def test_namespace_discovery_answers_for_the_requested_instant( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and download must agree, or the picker offers columns the download returns NA.""" + published = self._seeded(client, session, data_provider, data_files, datetime(2020, 1, 1, tzinfo=timezone.utc)) + variant_urn = _first_variant_urn(session, published["urn"]) + url = f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces" + + current = client.get(url) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert "vep" in {entry["namespace"] for entry in current.json()} + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(url, params={"as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert "vep" not in {entry["namespace"] for entry in before.json()} + + class TestCsvNamespaceDiscovery: """The discovery endpoints advertise what a namespace picker should offer.""" def test_score_set_namespaces_are_labeled_and_grouped( self, session, data_provider, client, setup_router_db, data_files ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") @@ -247,7 +529,7 @@ def test_score_set_namespaces_are_labeled_and_grouped( def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): """Discovery is its own request, so it must not appear on the score-set page's critical path.""" - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) response = client.get(f"/api/v1/score-sets/{published['urn']}") @@ -257,7 +539,7 @@ def test_score_set_detail_response_is_unchanged(self, session, data_provider, cl def test_variant_namespaces_are_labeled_and_grouped( self, session, data_provider, client, setup_router_db, data_files ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") @@ -273,10 +555,10 @@ def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( ): """Discovery and validation must agree, or the picker offers options that 422.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + link_clinical_controls_to_alleles(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py new file mode 100644 index 000000000..54d7fadf8 --- /dev/null +++ b/tests/routers/test_variant_annotation.py @@ -0,0 +1,352 @@ +# ruff: noqa: E402 +"""Router tests for the variant annotation surface relocated onto the Allele substrate. + +The VA-Spec statement endpoints (``/variants/{urn}/va/*``) and the VRS-identifier lookup +(``/variants/vrs/{identifier}``) moved off the legacy ``/mapped-variants`` router in #743; they build +from the ``MappingRecord`` / ``Allele`` substrate (seeded by ``seed_annotation_substrate``), never the +legacy ``MappedVariant``. +""" + +import json +from datetime import datetime, timezone + +import pytest + +from tests.helpers.util.user import change_ownership + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote_plus + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +from sqlalchemy import select + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_GA4GH_IDENTIFIER, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + create_seq_score_set_with_variants, + seed_annotation_substrate, +) + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +# --- StudyResult ----------------------------------------------------------------------------------- + + +def test_variant_study_result(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." + ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) + + +def test_variant_study_result_404_when_variant_missing(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + urn = f"{score_set['urn']}#404" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert response.json()["detail"] == f"variant with URN '{urn}' not found" + + +def test_variant_study_result_404_when_no_mapping_data(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # The variant exists but has no mapping record seeded on the new substrate. + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert f"No study result exists for variant {urn}: no mapping data exists." in response.json()["detail"] + + +# --- Functional-impact Statement ------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_functional_impact_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." + Statement.model_validate_json(json.dumps(response_data)) + + +def test_variant_functional_impact_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # Mapped on the new substrate, but no (primary) calibration => insufficient evidence. + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/functional-statement") + + assert response.status_code == 404 + assert ( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- Pathogenicity Statement ----------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_pathogenicity_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") + response_data = response.json() + + assert response.status_code == 200 + assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] + VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) + + +def test_variant_pathogenicity_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/pathogenicity-statement") + + assert response.status_code == 404 + assert ( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- VRS-identifier lookup ------------------------------------------------------------------------- + + +def test_lookup_variants_by_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # vrs_digest stores the full GA4GH VRS id in production (the mapper writes post_mapped["id"] into it), + # so the lookup matches the identifier against vrs_digest. The allele's ClinGen id rides alongside the URN. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + response_data = response.json() + + assert response.status_code == 200 + assert len(response_data) == 1 + assert response_data[0]["variantUrn"] == f"{score_set['urn']}#1" + assert response_data[0]["clingenAlleleId"] == "CA1" + assert response_data[0]["vrsId"] == TEST_GA4GH_IDENTIFIER + + +def test_lookup_variants_by_vrs_identifier_empty_when_none_match( + client, session, data_provider, data_files, setup_router_db +): + """The lookup is collection-shaped: no match is an empty list, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + # A well-formed GA4GH id (32-char digest) that no seeded allele carries. + response = client.get(f"/api/v1/variants/vrs/{quote_plus('ga4gh:VA.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')}") + + assert response.status_code == 200 + assert response.json() == [] + + +def test_lookup_variants_by_vrs_identifier_empty_without_read_permission( + client, session, data_provider, data_files, setup_router_db +): + """Existence-hiding: an identifier that matches only a private score set is filtered to an empty list, + indistinguishable from a genuine no-match — so the response never reveals the private allele exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # The identifier matches a real allele (by vrs_digest); the permission filter drops it to an empty list. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [] + + +# --- as_of temporal behavior ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("va_endpoint", ["study-result", "functional-statement", "pathogenicity-statement"]) +def test_va_endpoint_404_when_no_mapping_data_at_as_of( + client, session, data_provider, data_files, setup_router_db, va_endpoint +): + """A past instant with no live mapping yields the single-resource 404 — the statement did not exist then. + + ``as_of`` is threaded into ``variant_annotation_context``, so a far-past instant sees no live mapping + record and the derived VA statement cannot be constructed. This is the deliberate contrast with the + score-set *collection* endpoints: an empty collection is 200 + empty, but a single derived resource + that does not exist at the requested instant is a 404. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/{va_endpoint}", params={"as_of": past.isoformat()}) + + assert response.status_code == 404 + assert "no mapping data exists" in response.json()["detail"] + + +def test_variant_study_result_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The study-result route reconstructs its subject at ``as_of`` and echoes the instant on ``X-As-Of``. + + Study result needs only live mapping (no calibration), so it cleanly exercises the positive path: a + far-future instant still sees the freshly-seeded substrate as live and returns 200. Omitting ``as_of`` + reports ``current``. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result" + + # No as_of: the header reports the current standing. + current_response = client.get(url) + assert current_response.status_code == 200 + assert current_response.headers["X-As-Of"] == "current" + + # Far future: the seeded mapping is live, so the subject reconstructs and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + +def test_lookup_variants_by_vrs_identifier_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The VRS lookup filters its matches by ``as_of`` (the allele link must be live at that instant).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + url = f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}" + + # Far future: the allele link is live, so the identifier resolves and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert len(future_response.json()) == 1 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + # Far past: nothing was live yet, so no allele matches — an empty collection, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.json() == [] diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index af77e8011..a6c47a44f 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -3,15 +3,21 @@ from datetime import date import pytest +from sqlalchemy import select from mavedb.models.acmg_classification import ACMGClassification -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.collection import Collection from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene @@ -280,14 +286,17 @@ def make_dump_score_set(session, sample_user, dump_experiment, dump_licenses, du Args: variant_scores: one score_data dict per variant. Keys become the score columns. count_columns: count column names; each variant gets a count_data value for every one. - mapped: attach a mapped variant to each variant. + mapped: give each variant a mapping — a live ``MappingRecord`` with an authoritative allele on + the allele substrate, plus a row in the frozen ``MappedVariant`` table, since the dump still + sources ``mapped-variants.json`` from the latter. current: whether those mappings are current. False models a fully superseded score set, which the dump must treat as having no annotations at all. - post_mapped: whether each mapping carries a post-mapped VRS allele. False leaves it NULL, which - is how a variant the mapper could not place is stored, and which the README documents as - yielding a null `annotation`. Note that the shared `TEST_MINIMAL_MAPPED_VARIANT` uses an - empty dict here, a shape production never stores and the annotation layer cannot parse. - published: sets published_date, which the dump's selection query requires. + placeable: whether the mapper placed each variant. False gives it a `MappingRecord` with no + allele link at all, which is how an unplaceable variant is stored — an `Allele`'s identity + is its post-mapped VRS id, so one cannot exist without a post-mapped representation. Do not + reintroduce a "linked allele with a NULL `post_mapped`" shape; no write path produces it. + published: publishes the score set — stamps `published_date`, which the dump's selection query + requires, and clears `private`, which permission checks read. Publishing sets both. cc0: whether the score set carries the CC0 license the dump requires. """ counter = {"n": 0} @@ -298,7 +307,7 @@ def _make( count_columns=(), mapped=True, current=True, - post_mapped=False, + placeable=True, published=True, cc0=True, ): @@ -318,6 +327,11 @@ def _make( modified_by=sample_user, licence_id=CC0_LICENSE_ID if cc0 else OTHER_LICENSE_ID, published_date=date(2024, 1, 1) if published else None, + # `ScoreSet.private` defaults to True, and publishing clears it alongside stamping + # `published_date` (see the publish route). Setting only the date would build a score set that + # is published *and* private — a state production never creates, and one that reads as + # unreadable to any permission check while still satisfying the dump's selection query. + private=not published, dataset_columns={"score_columns": score_columns, "count_columns": list(count_columns)}, target_genes=[ TargetGene( @@ -356,7 +370,7 @@ def _make( **{ **TEST_MINIMAL_MAPPED_VARIANT, "current": current, - "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped else None, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if placeable else None, }, variant_id=variant.id, clingen_allele_id=f"CA{counter['n']}{index}", @@ -364,6 +378,30 @@ def _make( ) session.commit() + # The annotation and CSV artifacts read the allele substrate, not the row above. An + # unplaceable variant gets the record and no allele, matching the `if post_mapped_allele:` + # guard in `worker/jobs/variant_processing/mapping.py`. + record = seed_mapping_record( + session, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest=f"dump-allele-{counter['n']}-{index}", + level="genomic", + is_authoritative=True, + clingen_allele_id=f"CA{counter['n']}{index}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ] + if placeable + else [], + ) + if not current: + # A fully superseded score set: the record is in history, but nothing is live. + record.retire(session) + session.commit() + session.refresh(score_set) return score_set @@ -372,20 +410,30 @@ def _make( @pytest.fixture def add_clinvar_control(session): - """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" - - def _add(mapped_variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): - mapped_variant.clinical_controls.append( - ClinicalControl( - db_identifier="183058", - gene_symbol="PTEN", - clinical_significance=significance, - clinical_review_status=review_status, - db_name="ClinVar", - db_version=db_version, - ) + """Attach a ClinVar clinical control to a variant's authoritative allele, for an `MM_YYYY` release.""" + + def _add(variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): + """Link a control to *variant*'s authoritative allele, which is where the CSV layer reads it.""" + allele = session.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant.id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, f"variant {variant.urn} has no authoritative allele to link a control to" + + control = ClinvarControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, ) - session.add(mapped_variant) + session.add(control) + session.commit() + session.add(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id)) session.commit() return _add diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py index 6ca5b459a..4c8919e15 100644 --- a/tests/scripts/test_export_public_data.py +++ b/tests/scripts/test_export_public_data.py @@ -40,6 +40,7 @@ score_set_has_current_mappings, scores_csv, va_ndjson, + vrs_ndjson, write_public_dump, ) from mavedb.view_models.experiment_set import ExperimentSetPublicDump @@ -53,6 +54,10 @@ def _header(csv_text): return next(csv.reader(io.StringIO(csv_text))) +def _variants_of(session, score_set): + return session.query(Variant).filter(Variant.score_set_id == score_set.id).order_by(Variant.id).all() + + def _mapped_variants_of(session, score_set): return ( session.query(MappedVariant) @@ -214,6 +219,7 @@ def test_a_mapped_score_set_contributes_every_annotation_artifact( assert set(artifacts) == { f"csv/{base}.scores.csv", f"csv/{base}.annotations.csv", + f"vrs/{base}.vrs.ndjson", f"mapped/{base}.mapped-variants.json", f"va/{base}.va.ndjson", } @@ -234,12 +240,12 @@ def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, a assert set(artifacts) == {f"csv/{base}.scores.csv", f"csv/{base}.counts.csv"} def test_every_path_sits_in_a_documented_directory(self, session, make_dump_score_set, anonymous_principal): - """README `Archive Structure` lists exactly these three directories.""" + """README `Archive Structure` lists exactly these directories.""" score_set = make_dump_score_set(count_columns=("c_0",)) artifacts = _artifacts(session, score_set, anonymous_principal) - assert {path.split("/")[0] for path in artifacts} == {"csv", "mapped", "va"} + assert {path.split("/")[0] for path in artifacts} == {"csv", "vrs", "mapped", "va"} def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score_set, anonymous_principal): """Truthiness alone would accept a header with no rows, which is the failure worth catching.""" @@ -256,7 +262,7 @@ def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score assert len(json.loads(content)) == 2, path def test_yields_each_artifact_without_holding_the_others(self, session, make_dump_score_set, anonymous_principal): - """A generator, so the caller writes and releases each payload rather than accumulating four. + """A generator, so the caller writes and releases each payload rather than accumulating them all. One score set's `va.ndjson` runs to tens of kilobytes per variant, so returning them together made peak memory the sum of a score set's artifacts instead of its largest one. @@ -268,7 +274,7 @@ def test_yields_each_artifact_without_holding_the_others(self, session, make_dum first_path, first_content = next(artifacts) assert first_path == f"csv/{archive_path_base(score_set.urn)}.scores.csv" assert first_content - assert sum(1 for _ in artifacts) == 4 # annotations, mapped, va, counts still unevaluated + assert sum(1 for _ in artifacts) == 5 # annotations, vrs, mapped, va, counts still unevaluated #################################################################################################### @@ -339,7 +345,7 @@ def test_carries_exactly_the_expected_groups( Both are legitimate changes to make. Neither should be possible without editing this list. """ score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) - add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + add_clinvar_control(_variants_of(session, score_set)[0], db_version="01_2024") calibration = make_dump_calibration(score_set) namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) @@ -358,9 +364,9 @@ def test_carries_every_ingested_clinvar_release( ): """README: this file carries every release MaveDB holds, not just the most recent.""" score_set = make_dump_score_set() - mapped_variant = _mapped_variants_of(session, score_set)[0] - add_clinvar_control(mapped_variant, db_version="01_2024") - add_clinvar_control(mapped_variant, db_version="06_2025") + variant = _variants_of(session, score_set)[0] + add_clinvar_control(variant, db_version="01_2024") + add_clinvar_control(variant, db_version="06_2025") namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) @@ -421,7 +427,7 @@ def test_a_clinvar_release_carries_its_release_in_the_column_name( self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer ): score_set = make_dump_score_set() - add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + add_clinvar_control(_variants_of(session, score_set)[0], db_version="01_2024") header = _header(annotations_csv(session, score_set, anonymous_viewer)) @@ -535,6 +541,52 @@ def test_is_a_json_array(self, session, make_dump_score_set): #################################################################################################### +def _vrs_records(session, score_set): + return [json.loads(line) for line in vrs_ndjson(session, score_set).splitlines()] + + +@pytest.mark.integration +class TestVrsNdjson: + """The GA4GH molecular objects, in one artifact rather than scattered across the archive.""" + + def test_emits_one_line_per_mapped_variant(self, session, make_dump_score_set): + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + content = vrs_ndjson(session, score_set) + + assert len(content.splitlines()) == 3 + + def test_every_line_is_newline_terminated(self, session, make_dump_score_set): + """A line-based consumer should need no special case for the last record.""" + content = vrs_ndjson(session, make_dump_score_set()) + + assert content.endswith("\n") + + def test_carries_the_vrs_pair_and_the_categorical_variant(self, session, make_dump_score_set): + record = _vrs_records(session, make_dump_score_set())[0] + + assert set(record) == {"variant_urn", "pre_mapped", "post_mapped", "categorical_variant"} + assert record["post_mapped"]["type"] == "Allele" + assert record["categorical_variant"]["type"] == "CategoricalVariant" + + def test_an_unplaceable_variant_is_omitted(self, session, make_dump_score_set): + """An unplaceable variant has a `MappingRecord` but no allele, and every field here is + allele-derived, so it contributes no line rather than a line of nulls.""" + assert vrs_ndjson(session, make_dump_score_set(placeable=False)) == "" + + def test_a_score_set_with_no_mappings_yields_nothing(self, session, make_dump_score_set): + assert vrs_ndjson(session, make_dump_score_set(mapped=False)) == "" + + def test_is_not_withheld_for_a_private_calibration(self, session, make_dump_score_set, make_dump_calibration): + """None of this is calibration-derived, so calibration visibility must not gate it.""" + score_set = make_dump_score_set() + make_dump_calibration(score_set, private=True) + + record = _vrs_records(session, score_set)[0] + + assert record["post_mapped"] is not None + + def _va_records(session, score_set, principal): return [json.loads(line) for line in va_ndjson(session, score_set, principal).splitlines()] @@ -559,28 +611,23 @@ def test_every_line_is_newline_terminated(self, session, make_dump_score_set, an assert "\n\n" not in content def test_every_line_is_an_envelope_with_both_fields(self, session, make_dump_score_set, anonymous_principal): - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": 1.0}, {"score": 2.0})) + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) records = _va_records(session, score_set, anonymous_principal) assert all(set(record) == {"variant_urn", "annotation"} for record in records) - def test_an_unannotatable_variant_still_gets_its_urn(self, session, make_dump_score_set, anonymous_principal): - """README: `annotation` is null for a current mapping with no post-mapped allele.""" - score_set = make_dump_score_set(post_mapped=False) - - records = _va_records(session, score_set, anonymous_principal) - - assert records - assert all(record["variant_urn"] for record in records) - assert all(record["annotation"] is None for record in records) + def test_an_unplaceable_variant_is_omitted(self, session, make_dump_score_set, anonymous_principal): + """An unplaceable variant carries no allele, so it is not an annotatable variant and + contributes no line.""" + assert va_ndjson(session, make_dump_score_set(placeable=False), anonymous_principal) == "" def test_is_empty_for_a_score_set_with_no_current_mappings(self, session, make_dump_score_set, anonymous_principal): assert va_ndjson(session, make_dump_score_set(mapped=False), anonymous_principal) == "" def test_a_post_mapped_variant_carries_an_annotation(self, session, make_dump_score_set, anonymous_principal): """The counterpart to the null case: a post-mapped allele is what makes a variant annotatable.""" - score_set = make_dump_score_set(post_mapped=True) + score_set = make_dump_score_set() records = _va_records(session, score_set, anonymous_principal) @@ -591,7 +638,7 @@ def test_a_clinical_calibration_raises_the_record_to_the_pathogenicity_layer( self, session, make_dump_score_set, make_dump_calibration, anonymous_principal ): """README `va/{urn}.va.ndjson`: the highest materialized layer, which a calibration supplies.""" - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, research_use_only=False) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -607,7 +654,7 @@ def test_a_research_use_only_calibration_does_not_raise_the_layer( The record falls back to the functional-impact layer rather than disappearing, so the variant is still reported — just without a pathogenicity statement built on evidence not cleared for it. """ - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, research_use_only=True) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -618,7 +665,7 @@ def test_a_private_calibration_does_not_raise_the_layer( self, session, make_dump_score_set, make_dump_calibration, anonymous_principal ): """A private calibration is withheld from every artifact, this one included.""" - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, private=True) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -648,7 +695,7 @@ def test_keeps_a_calibration_named_visible(self, session, make_dump_score_set, m session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - narrowed = public_experiment_set(view, {calibration.id}) + narrowed = public_experiment_set(view, {calibration.id}, set()) kept = narrowed.experiments[0].score_sets[0].score_calibrations assert [c.id for c in kept] == [calibration.id] @@ -659,7 +706,7 @@ def test_drops_a_calibration_not_named_visible(self, session, make_dump_score_se session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - narrowed = public_experiment_set(view, set()) + narrowed = public_experiment_set(view, set(), set()) assert narrowed.experiments[0].score_sets[0].score_calibrations == [] @@ -673,7 +720,49 @@ def test_returns_none_when_no_experiment_has_a_score_set(self, experiment_set_vi } ) - assert public_experiment_set(emptied, set()) is None + assert public_experiment_set(emptied, set(), set()) is None + + def _superseded_view(self, session, score_set, successor, readable_superseding_urns): + score_set.superseding_score_set = successor + session.commit() + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).first()) + + narrowed = public_experiment_set(view, set(), readable_superseding_urns) + return next(ss for e in narrowed.experiments for ss in e.score_sets if ss.urn == str(score_set.urn)) + + def test_blanks_a_supersession_the_caller_may_not_read(self, session, make_dump_score_set): + """A published score set is usually superseded by a private in-progress replacement. + + `superseding_score_set` is inherited from `SavedScoreSet` and hydrated from the ORM relationship + during validation, so without narrowing it the archive names an unreleased score set's URN and + title in `main.json` — published to Zenodo, where it cannot be recalled. + """ + superseded = self._superseded_view(session, make_dump_score_set(), make_dump_score_set(published=False), set()) + + assert superseded.superseding_score_set is None + + def test_keeps_a_supersession_the_caller_may_read(self, session, make_dump_score_set): + """The blank withholds only what the caller cannot already see.""" + successor = make_dump_score_set() + superseded = self._superseded_view(session, make_dump_score_set(), successor, {str(successor.urn)}) + + assert superseded.superseding_score_set is not None + assert superseded.superseding_score_set.urn == str(successor.urn) + + def test_keeps_a_readable_supersession_whose_data_the_archive_does_not_carry(self, session, make_dump_score_set): + """Gated on READ, not on archive membership. + + A published successor under a non-CC0 license is public knowledge; its data living outside this + archive is no reason to hide that the score set it replaced is superseded. Suppressing the pointer + would leave a consumer treating a superseded score set as current, which is the worse failure. The + successor is deliberately absent from the archive here, and must still be named. + """ + successor = make_dump_score_set(cc0=False) + superseded = self._superseded_view(session, make_dump_score_set(), successor, {str(successor.urn)}) + + assert superseded.superseding_score_set is not None + assert superseded.superseding_score_set.urn == str(successor.urn) def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_dump_calibration): """`ScoreSet.score_calibrations` cascades delete-orphan, so narrowing the ORM collection instead @@ -685,7 +774,7 @@ def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_ session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - public_experiment_set(view, set()) + public_experiment_set(view, set(), set()) session.commit() assert session.query(ScoreCalibration).filter(ScoreCalibration.id == calibration.id).one_or_none() @@ -754,6 +843,16 @@ def test_separates_the_two_within_one_experiment(self, session, make_dump_score_ #################################################################################################### +def _metadata_supersessions(metadata): + """Every score set's reported superseding URN, keyed by its own URN.""" + return { + score_set.urn: (score_set.superseding_score_set.urn if score_set.superseding_score_set else None) + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + + def _metadata_calibration_ids(metadata): return { calibration.id @@ -817,6 +916,45 @@ def test_carries_a_research_use_only_calibration( assert _metadata_calibration_ids(metadata) == {calibration_id} + @pytest.mark.parametrize( + "successor_published,successor_cc0,expect_named", + [ + (True, True, True), # a successor the archive also carries + (True, False, False), # published, but its data is not in the archive + (False, True, False), # an in-progress replacement + (False, False, False), + ], + ) + def test_supersession_is_named_only_when_the_archive_carries_the_successor( + self, session, make_dump_score_set, anonymous_principal, successor_published, successor_cc0, expect_named + ): + """What `main.json` says about supersession, and why. + + `published_experiment_sets` loads members through `lazyload(...).and_(published + CC0)`, and that + criteria propagates to the self-referential `superseding_score_set` relationship. So a successor + hydrates exactly when it satisfies the member filter itself — the relationship is fine, an + unfiltered query finds every one of these. + + Two consequences worth pinning: + + - A private, in-progress successor is never named, so the narrowing in `public_experiment_set` is + defence in depth against this path rather than a live gate. + - A *published* successor whose license keeps its data out of the archive is not named either. That + one is a real gap: the score set is public knowledge, and a consumer is left reading a superseded + score set as current. Closing it needs the successor fetched by a separate unfiltered query and + injected into the view, not merely left un-blanked. + """ + superseded = make_dump_score_set() + successor = make_dump_score_set(published=successor_published, cc0=successor_cc0) + superseded.superseding_score_set = successor + session.commit() + urn, successor_urn = superseded.urn, successor.urn + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_supersessions(metadata)[urn] == (successor_urn if expect_named else None) + def test_reports_the_urns_whose_artifacts_the_archive_carries( self, session, make_dump_score_set, anonymous_principal ): diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index eae324466..f2d1f2722 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -11,8 +11,8 @@ from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline from mavedb.scripts.run_score_set_pipelines import ( - CLUSTER_KEY_UNKNOWN, _IN_FLIGHT_STATUSES, + CLUSTER_KEY_UNKNOWN, cluster_cohort, cohort_filename, effective_pipeline_name, @@ -311,7 +311,11 @@ class TestResolveJobSubset: def test_caid_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"submit_score_set_mappings_to_car"})) - assert {j["key"] for j in subset} == {"map_variants_for_score_set", "submit_score_set_mappings_to_car"} + assert {j["key"] for j in subset} == { + "reverse_translate_variants_for_score_set", + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + } def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] @@ -319,21 +323,18 @@ def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } ) subset = resolve_job_subset(jobs, leaf) assert {j["key"] for j in subset} == { + "reverse_translate_variants_for_score_set", "map_variants_for_score_set", "submit_score_set_mappings_to_car", "warm_clingen_cache", "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } @@ -357,8 +358,6 @@ def test_vep_leaf_resolves_against_map_annotate_score_set(self): { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } diff --git a/tests/worker/conftest_optional.py b/tests/worker/conftest_optional.py index 0f1d2e95f..ccd4208a0 100644 --- a/tests/worker/conftest_optional.py +++ b/tests/worker/conftest_optional.py @@ -4,6 +4,7 @@ import pytest from arq import ArqRedis from cdot.hgvs.dataproviders import RESTDataProvider +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from sqlalchemy.orm import Session from mavedb.worker.lib.managers.job_manager import JobManager @@ -52,6 +53,7 @@ def mock_worker_ctx(): """Create a mock worker context dictionary for testing.""" mock_redis = Mock(spec=ArqRedis) mock_hdp = Mock(spec=RESTDataProvider) + mock_seqrepo = Mock(spec=SeqRepoDataProxy) mock_pool = Mock(spec=ProcessPoolExecutor) # Don't mock the session itself to allow real DB interactions in tests @@ -60,5 +62,6 @@ def mock_worker_ctx(): return { "redis": mock_redis, "hdp": mock_hdp, + "seqrepo": mock_seqrepo, "pool": mock_pool, } diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index de835a56d..873d09ff0 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1,9 +1,13 @@ import pytest +from sqlalchemy import select +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import DependencyType from mavedb.models.job_dependency import JobDependency from mavedb.models.job_run import JobRun from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -44,6 +48,38 @@ def map_variants_sample_params(with_populated_domain_data, sample_score_set, sam } +@pytest.fixture +def reverse_translate_variants_sample_params(with_populated_domain_data, sample_score_set): + """Provide sample parameters for reverse_translate_variants_for_score_set job.""" + + return { + "score_set_id": sample_score_set.id, + "correlation_id": "sample-reverse-translation-correlation-id", + } + + +@pytest.fixture +def sample_independent_reverse_translation_run(reverse_translate_variants_sample_params): + """Create a JobRun instance for the reverse_translate_variants_for_score_set job.""" + + return JobRun( + urn="test:reverse_translate_variants_for_score_set", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=reverse_translate_variants_sample_params, + ) + + +@pytest.fixture +def with_reverse_translation_run(session, sample_independent_reverse_translation_run): + """Add a reverse_translate_variants_for_score_set job run to the session.""" + + session.add(sample_independent_reverse_translation_run) + session.commit() + + @pytest.fixture def link_gnomad_variants_sample_params(with_populated_domain_data, sample_score_set): """Provide sample parameters for create_variants_for_score_set job.""" @@ -249,6 +285,87 @@ def setup_sample_variants_with_caid( return variant, mapped_variant +@pytest.fixture +def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_link_gnomad_variants_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the gnomAD linkage job. The allele carries the CAID matched by the mocked Athena row, and the + allele is the authoritative measurement for the variant so the bandaid seam writes its VAS row. + """ + score_set = session.get(ScoreSet, sample_link_gnomad_variants_run.job_params["score_set_id"]) + + variant = Variant( + urn="urn:variant:test-variant-with-allele-caid", + score_set_id=score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, + ) + allele = Allele( + vrs_digest="test-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NM_000000.1:c.1A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) + session.commit() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="pytest.0.0", + ) + session.add(mapping_record) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_with_caid(session, setup_sample_alleles_with_caid): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying + the CAID the mocked gnomAD row matches. The authoritative allele is given a CAID with no gnomAD + match, so only the RT-derived allele can link. This isolates the requirement that gnomAD linkage + must cover the full allele set (authoritative + RT-derived), not just authoritative links — for + protein/coding score sets the genomic allele gnomAD knows is the RT-derived one. + """ + variant, authoritative_allele = setup_sample_alleles_with_caid + + # Authoritative allele's CAID intentionally has no gnomAD match, so it cannot be what links. + authoritative_allele.clingen_allele_id = "CA_NO_GNOMAD_MATCH" + session.add(authoritative_allele) + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000001.11:g.12345G>A", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele + + ## Uniprot Job Fixtures ## @@ -894,198 +1011,6 @@ def with_cleanup_job(session, sample_cleanup_job_run): session.commit() -## HGVS Population Job Fixtures ## - - -@pytest.fixture -def populate_hgvs_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_hgvs_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_hgvs_pipeline(): - """Create a pipeline instance for populate_hgvs_for_score_set job.""" - - return Pipeline( - urn="test:populate_hgvs_pipeline", - name="Populate HGVS Pipeline", - ) - - -@pytest.fixture -def sample_populate_hgvs_run(populate_hgvs_sample_params): - """Create a JobRun instance for populate_hgvs_for_score_set job.""" - - return JobRun( - urn="test:populate_hgvs_for_score_set", - job_type="populate_hgvs_for_score_set", - job_function="populate_hgvs_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_hgvs_sample_params, - ) - - -@pytest.fixture -def with_populate_hgvs_job(session, sample_populate_hgvs_run): - """Add a populate_hgvs_for_score_set job run to the session.""" - - session.add(sample_populate_hgvs_run) - session.commit() - - -@pytest.fixture -def with_populate_hgvs_pipeline(session, sample_populate_hgvs_pipeline): - """Add a populate_hgvs pipeline to the session.""" - - session.add(sample_populate_hgvs_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_hgvs_run_pipeline( - session, - with_populate_hgvs_job, - with_populate_hgvs_pipeline, - sample_populate_hgvs_run, - sample_populate_hgvs_pipeline, -): - """Provide a context with a populate_hgvs job run and pipeline.""" - - sample_populate_hgvs_run.pipeline_id = sample_populate_hgvs_pipeline.id - session.commit() - return sample_populate_hgvs_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_hgvs( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_hgvs_run -): - """Setup variants and mapped variants in the database for HGVS population testing.""" - score_set = session.get(ScoreSet, sample_populate_hgvs_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-hgvs", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - -# --- Variant Translation Fixtures --- - - -@pytest.fixture -def populate_variant_translations_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_variant_translations_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_variant_translations_pipeline(): - """Create a pipeline instance for populate_variant_translations_for_score_set job.""" - - return Pipeline( - urn="test:populate_variant_translations_pipeline", - name="Populate Variant Translations Pipeline", - ) - - -@pytest.fixture -def sample_populate_variant_translations_run(populate_variant_translations_sample_params): - """Create a JobRun instance for populate_variant_translations_for_score_set job.""" - - return JobRun( - urn="test:populate_variant_translations_for_score_set", - job_type="populate_variant_translations_for_score_set", - job_function="populate_variant_translations_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_variant_translations_sample_params, - ) - - -@pytest.fixture -def with_populate_variant_translations_job(session, sample_populate_variant_translations_run): - """Add a populate_variant_translations_for_score_set job run to the session.""" - - session.add(sample_populate_variant_translations_run) - session.commit() - - -@pytest.fixture -def with_populate_variant_translations_pipeline(session, sample_populate_variant_translations_pipeline): - """Add a populate_variant_translations pipeline to the session.""" - - session.add(sample_populate_variant_translations_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_variant_translations_run_pipeline( - session, - with_populate_variant_translations_job, - with_populate_variant_translations_pipeline, - sample_populate_variant_translations_run, - sample_populate_variant_translations_pipeline, -): - """Provide a context with a populate_variant_translations job run and pipeline.""" - - sample_populate_variant_translations_run.pipeline_id = sample_populate_variant_translations_pipeline.id - session.commit() - return sample_populate_variant_translations_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_translation( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_variant_translations_run -): - """Setup variants and mapped variants in the database for variant translation testing.""" - score_set = session.get(ScoreSet, sample_populate_variant_translations_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-translation", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - ## ClinGen Cache Warming Job Fixtures ## @@ -1223,8 +1148,14 @@ def sample_populate_vep_run_pipeline( @pytest.fixture -def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a variant and mapped variant with hgvs_assay_level for VEP testing.""" +def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the VEP consequence job. The allele carries an HGVS the job submits to VEP and is the + authoritative measurement for the variant, so the bandaid seam writes its per-variant VAS row. + + The HGVS lives on ``Allele.hgvs_c`` (a coding HGVS VEP resolves directly) — the new job reads its + submission string from the allele, not from ``MappedVariant.hgvs_assay_level``. + """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( @@ -1234,28 +1165,42 @@ def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_work hgvs_pro="NP_009225.1:p.Cys2Tyr", data={"hgvs_c": "NM_007294.4:c.5A>G", "hgvs_p": "NP_009225.1:p.Cys2Tyr"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-allele-vrs-digest", + level="cdna", + hgvs_c="NM_007294.4:c.5A>G", + post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, - hgvs_assay_level="NM_007294.4:c.5A>G", + assay_level="cdna", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele @pytest.fixture -def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a protein HGVS variant (NP_ accession) that VEP cannot resolve directly. +def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up an allele whose only HGVS is a protein HGVS (NP_ accession) that VEP cannot resolve + directly. - VEP's /vep/human/hgvs endpoint does not return results for protein HGVS strings like - NP_009225.1:p.Val1696His, so these must be recoded via Variant Recoder first. This fixture - exercises the recoder fallback path end-to-end. + VEP's /vep/human/hgvs endpoint returns no consequence for protein HGVS like + NP_009225.1:p.Val1696His, so the job must fall back to Variant Recoder. ``hgvs_g``/``hgvs_c`` are + left unset so the VEP payload resolves to ``hgvs_p``, exercising the recoder fallback path. """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) @@ -1265,17 +1210,61 @@ def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mo hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-protein-allele-vrs-digest", + level="protein", + hgvs_p="NP_009225.1:p.Val1696His", + post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, - hgvs_assay_level="NP_009225.1:p.Val1696His", + assay_level="protein", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_for_vep(session, setup_sample_alleles_for_vep): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying a + genomic HGVS of its own. Isolates the requirement that VEP linkage covers the full allele set + (authoritative + RT-derived), while the per-variant VAS fan-out stays authoritative-only. + """ + variant, authoritative_allele = setup_sample_alleles_for_vep + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-vep-allele-vrs-digest", + level="genomic", + hgvs_g="NC_000017.11:g.43124027T>C", + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000017.11:g.43124027T>C", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index f29842158..875eb635b 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,11 +10,12 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.clinvar import generate_clinvar_versions +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -43,7 +44,7 @@ async def test_refresh_clinvar_controls_e2e( arq_redis, arq_worker, standalone_worker_context, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): @@ -53,7 +54,7 @@ async def test_refresh_clinvar_controls_e2e( """ with ( patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=_E2E_VERSIONS, ), patch( @@ -65,27 +66,25 @@ async def test_refresh_clinvar_controls_e2e( await arq_worker.async_run() await arq_worker.run_check() - # The variant should be present in the 2025-01 archive. - clinical_controls = session.scalars(select(ClinicalControl)).all() + # Verify that clinical controls were added successfully — one row per ClinVar version + # that contains the variant, so there may be more than one. + clinical_controls = session.scalars(select(ClinvarControl)).all() assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - VariantAnnotationStatus.status == AnnotationStatus.SUCCESS, + # Verify that at least one present event was recorded for the allele. The job processes one + # event per ClinVar version; versions without the allele produce absent events, so filtering + # for present gives a stable assertion. Events are allele-keyed (no variant_id). + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.PRESENT, ) ).all() - assert len(success_annotations) == 1 - - # Total annotations == versions processed. - all_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - ) - ).all() - assert len(all_annotations) == len(_E2E_VERSIONS) + assert len(present_events) >= 1 + assert all(e.variant_id is None and e.allele_id is not None for e in present_events) + # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED @@ -101,7 +100,7 @@ async def test_latest_clinvar_archive_has_expected_schema(self): CLINVAR_FIELDS_TO_KEEP is present. Fails fast if ClinVar renames or removes a column before a deploy reaches production. """ - year, month = generate_clinvar_versions()[-1] + year, month = _generate_clinvar_versions()[-1] data = await fetch_clinvar_variant_data(month, year) assert len(data) > 0, "ClinVar archive returned no records" diff --git a/tests/worker/jobs/external_services/network/test_hgvs.py b/tests/worker/jobs/external_services/network/test_hgvs.py deleted file mode 100644 index 56f100e0b..000000000 --- a/tests/worker/jobs/external_services/network/test_hgvs.py +++ /dev/null @@ -1,54 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateHgvsForScoreSet: - """End-to-end test for HGVS population against the real ClinGen API.""" - - async def test_populate_hgvs_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Enqueue the HGVS population job, run the worker, and verify HGVS fields are populated.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - session.refresh(mapped_variant) - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.MAPPED_HGVS, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_variant_translations.py b/tests/worker/jobs/external_services/network/test_variant_translations.py deleted file mode 100644 index b45087dd9..000000000 --- a/tests/worker/jobs/external_services/network/test_variant_translations.py +++ /dev/null @@ -1,56 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateVariantTranslationsForScoreSet: - """End-to-end test for variant translation population against the real ClinGen API.""" - - async def test_populate_variant_translations_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Enqueue the variant translation job, run the worker, and verify translations are created.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VARIANT_TRANSLATION, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 61071fc42..61ba8dfd4 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -7,8 +7,10 @@ from sqlalchemy import select from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -27,10 +29,10 @@ async def test_populate_vep_e2e( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Enqueue the VEP job, run the worker, and verify consequence and annotation are populated.""" - _, mapped_variant = setup_sample_variants_for_vep + """Enqueue the VEP job, run the worker, and verify the allele's consequence and annotation.""" + variant, allele = setup_sample_alleles_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -42,18 +44,23 @@ async def test_populate_vep_e2e( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None + + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT async def test_populate_vep_e2e_with_recoder_path( self, @@ -62,7 +69,7 @@ async def test_populate_vep_e2e_with_recoder_path( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_protein_variant_for_vep, + setup_sample_protein_allele_for_vep, ): """VEP job uses Variant Recoder for a protein HGVS (NP_ accession) that VEP cannot resolve directly. @@ -70,7 +77,7 @@ async def test_populate_vep_e2e_with_recoder_path( does not return a consequence for. The job must fall back to Variant Recoder, recode it to a genomic HGVS, and then re-query VEP with the recoded string. """ - _, mapped_variant = setup_sample_protein_variant_for_vep + variant, allele = setup_sample_protein_allele_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -82,15 +89,20 @@ async def test_populate_vep_e2e_with_recoder_path( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None + + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 616263dfa..5431227d6 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -5,16 +5,18 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from copy import deepcopy from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clingen import ( submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -51,9 +53,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_mappings( self, @@ -76,9 +78,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( self, @@ -101,9 +103,9 @@ async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_registered_alleles( self, @@ -147,18 +149,18 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 variants dedup to 1 allele → one allele-keyed event, failed (no CAR response). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_all_car_errors( self, @@ -184,18 +186,18 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( dummy_variant_mapping_job_run, ) - # Build error responses for every submitted HGVS — CAR explicitly rejected all of them - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele (same VRS digest). Build an error response for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "errorType": "InvalidHGVS", - "hgvs": get_hgvs_from_post_mapped(mv.post_mapped) or "", + "hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", "message": "Invalid HGVS expression.", "description": "", - "inputLine": get_hgvs_from_post_mapped(mv.post_mapped) or "", - "position": str(i), + "inputLine": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", + "position": "0", } - for i, mv in enumerate(mapped_variants) ] with ( @@ -215,20 +217,105 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele, rejected by CAR → one failed event (reason=service_rejected). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "service_rejected" - async def test_submit_score_set_mappings_to_car_repeated_hgvs( + async def test_submit_score_set_mappings_to_car_event_per_allele( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Each allele — authoritative and derived — gets exactly one allele-keyed event; there is no + per-variant fan-out. Both alleles are registered with a CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # The sample's 4 variants dedup to one authoritative allele. + authoritative_allele = session.scalars(select(Allele)).one() + + # Attach a second, derived (is_authoritative=False) allele to one variant's current mapping + # record. It shares the authoritative allele's HGVS (different vrs_digest) so it is submitted + # under the same line — the realistic shape is a different level, but identical HGVS is enough + # to exercise the multi-allele-per-variant fan-out. + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + derived_allele = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-derived", + level=authoritative_allele.level, + post_mapped={**deepcopy(authoritative_allele.post_mapped), "id": "derived-allele-id"}, + ) + session.add(derived_allele) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived_allele.id, + is_authoritative=False, + ) + ) + session.flush() + + def fake_dispatch(hgvs_list): + return [{"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": []} for idx, _ in enumerate(hgvs_list)] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # One event per allele (2 alleles → 2 events), each keyed on its allele, not fanned per-variant. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 2 + assert all(e.variant_id is None for e in events) + assert {e.allele_id for e in events} == {authoritative_allele.id, derived_allele.id} + assert all(e.disposition == "present" for e in events) + + # Both alleles registered: registration breadth covers the derived allele too. + session.refresh(authoritative_allele) + session.refresh(derived_allele) + assert authoritative_allele.clingen_allele_id is not None + assert derived_allele.clingen_allele_id is not None + + async def test_submit_score_set_mappings_to_car_response_count_mismatch( self, mock_worker_ctx, session, @@ -252,14 +339,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles with repeated HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele → 1 submitted HGVS, but CAR returns 2 results. The count + # violates the one-result-per-input contract, so positional alignment can't be trusted + # and the whole batch must be rejected without writing any CAID. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ - { - "@id": "CA_DUPLICATE", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mapped_variants[0].post_mapped)}], - } + {"@id": "CA111111", "type": "nucleotide", "genomicAlleles": []}, + {"@id": "CA222222", "type": "nucleotide", "genomicAlleles": []}, ] with ( @@ -267,10 +354,62 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to return the same HGVS for all variants + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID written — neither of the returned values is trusted. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "api_error" + + async def test_submit_score_set_mappings_to_car_malformed_response( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # CAR returns a response that is neither an error (no "errorType") nor a registration + # (no "@id"). This must be surfaced as a rejection, not crash the loop with a KeyError. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [{"type": "nucleotide", "genomicAlleles": []}] + + with ( patch( - "mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", - return_value=get_hgvs_from_post_mapped(mapped_variants[0].post_mapped), + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), @@ -282,24 +421,19 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( ) assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == "CA_DUPLICATE" + assert result.status == JobStatus.FAILED - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # No CAID assigned; the one allele's event is failed (reason=malformed_response). + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "malformed_response" - async def test_submit_score_set_mappings_to_car_partial_failure( + async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, mock_worker_ctx, session, @@ -312,7 +446,6 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures (some matched, some not) result in a succeeded outcome with failure annotations.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -324,16 +457,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Get mapped variants; return a CAR response that only matches the first variant - mapped_variants = session.scalars(select(MappedVariant)).all() - assert len(mapped_variants) == 4 - - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + # 4 variants share 1 allele; CAR returns 1 response for the 1 submitted HGVS + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": "CA_DUPLICATE", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -353,33 +484,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 - # Verify only the first variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele received the CAID + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 1 + assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # Verify annotation statuses: 1 success, 3 failed - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", - ) + # 1 allele → one present event. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) - ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None - async def test_submit_score_set_mappings_to_car_hgvs_not_found( + async def test_submit_score_set_mappings_to_car_partial_failure( self, mock_worker_ctx, session, @@ -392,6 +512,7 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -403,21 +524,15 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant) - .join(Variant) - .where(Variant.score_set_id == submit_score_set_mappings_to_car_sample_job_run.job_params["score_set_id"]) - ).all() + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -425,8 +540,6 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to not find any HGVS in registered alleles - patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), ): @@ -436,21 +549,76 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), ) + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 + + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" + + # All 4 variant annotations succeeded + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", + ) + ).all() + assert len(present_events) == 1 + + async def test_submit_score_set_mappings_to_car_hgvs_not_found( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Patch get_hgvs_from_post_mapped to return None for all alleles + with ( + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "not_applicable" + assert event.reason == "no_hgvs" async def test_submit_score_set_mappings_to_car_propagates_exception( self, @@ -519,20 +687,16 @@ async def test_submit_score_set_mappings_to_car_success( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant).join(Variant).where(Variant.score_set_id == sample_score_set.id) - ).all() - assert len(mapped_variants) == 4 + # All 4 variants share 1 allele (same VRS digest from construct_mock_mapping_output) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -552,20 +716,371 @@ async def test_submit_score_set_mappings_to_car_success( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None + + async def test_submit_score_set_mappings_to_car_preexisting( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Already-registered allele is re-annotated as preexisting, not re-submitted.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Pre-set the CAID on the allele to simulate prior registration + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_PRIOR" + session.flush() + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + ) as mock_dispatch, + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # CAR should NOT be called since the allele is already registered + mock_dispatch.assert_not_called() + assert result.status == JobStatus.SUCCEEDED + assert result.data["already_registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 0 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "preexisting" + assert event.event_metadata["clingen_allele_id"] == "CA_PRIOR" + + async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration that returns the same CAID is a success with registration_source=reconfirmed.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_CONFIRMED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + registered_alleles_mock = [{"@id": "CA_CONFIRMED", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 1 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "reconfirmed" + + async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration returning a different CAID fails without overwriting the stored CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_STORED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + # CAR returns a DIFFERENT CAID — this is an invariant violation + registered_alleles_mock = [{"@id": "CA_DIFFERENT", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # Job fails because all CAID-returning submissions had a conflict + assert result.status == JobStatus.FAILED + + # CAID must NOT have been overwritten + session.refresh(allele) + assert allele.clingen_allele_id == "CA_STORED" + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "caid_conflict" + assert event.event_metadata["clingen_allele_id"] == "CA_STORED" + assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" + + async def _add_derived_alleles_with_distinct_hgvs(self, session, count): + """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR + job produces ``count + 1`` distinct submission lines (one per HGVS). Returns every allele.""" + authoritative_allele = session.scalars(select(Allele)).one() + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + + alleles = [authoritative_allele] + for i in range(count): + post_mapped = deepcopy(authoritative_allele.post_mapped) + post_mapped["expressions"][0]["value"] = f"NC_000001.11:g.{1000 + i}A>T" + derived = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-d{i}", + level=authoritative_allele.level, + post_mapped=post_mapped, + ) + session.add(derived) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived.id, + is_authoritative=False, + ) + ) + alleles.append(derived) + + session.flush() + return alleles + + async def test_submit_score_set_mappings_to_car_batches_submissions( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A large allele set is dispatched in fixed-size chunks rather than one PUT: 4 distinct HGVS + with a batch size of 2 yields two dispatch calls, and every allele is still registered.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + alleles = await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + dispatched_batches = [] + + def fake_dispatch(hgvs_list): + dispatched_batches.append(list(hgvs_list)) + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # 4 distinct HGVS at batch size 2 → two dispatch calls, neither exceeding the batch size, and + # together covering every submitted line exactly once. + assert len(dispatched_batches) == 2 + assert all(len(batch) <= 2 for batch in dispatched_batches) + assert sum(len(batch) for batch in dispatched_batches) == 4 + + # Every allele across both batches is registered. + for allele in alleles: + session.refresh(allele) + assert allele.clingen_allele_id is not None + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert all(event.disposition == "present" for event in events) + + async def test_submit_score_set_mappings_to_car_batch_failure_isolated( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A failed batch fails only its own alleles: the first chunk returns nothing (request failure) + while the second registers normally. Per-batch reconciliation keeps the two independent.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + call_count = {"n": 0} + + def fake_dispatch(hgvs_list): + call_count["n"] += 1 + # First batch mimics a request failure (dispatch_submissions returns [] on error); the rest + # register normally. + if call_count["n"] == 1: + return [] + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # The good batch links alleles, so the job overall succeeds despite the failed batch. + assert result.status == JobStatus.SUCCEEDED + assert call_count["n"] == 2 + + # Exactly the second batch's two alleles are registered; the first batch's two are not. + registered = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(registered) == 2 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert sum(event.disposition == "present" for event in events) == 2 + failed = [event for event in events if event.disposition == "failed"] + assert len(failed) == 2 + assert all(event.reason == "api_error" for event in failed) @pytest.mark.integration @@ -597,15 +1112,14 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -623,19 +1137,18 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -666,15 +1179,14 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -692,19 +1204,18 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run_in_pipeline) @@ -738,13 +1249,13 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -777,13 +1288,13 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -808,13 +1319,13 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -862,15 +1373,15 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -903,9 +1414,6 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( # Patch ClinGenAlleleRegistryService to return only errors with no linked alleles registered_alleles_mock = [ {"errorType": "InvalidHGVS", "hgvs": "test"}, - {"errorType": "InvalidHGVS", "hgvs": "test2"}, - {"errorType": "InvalidHGVS", "hgvs": "test3"}, - {"errorType": "InvalidHGVS", "hgvs": "test4"}, ] with ( @@ -925,15 +1433,15 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -952,7 +1460,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures result in SUCCEEDED status with per-variant failure annotations committed.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed, job SUCCEEDED.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -964,14 +1472,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Return a CAR response that only matches the first variant's HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -991,31 +1499,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( mock_send_slack_error.assert_not_called() assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 - - # Verify the successfully matched variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" - - # Verify annotation statuses: 1 success, 3 failed - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", - ) - ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 + + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" + + # All 4 variant annotations succeeded + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(present_events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,22 +1545,17 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno dummy_variant_mapping_job_run, ) - # Return a CAR response where: first variant succeeds, second has explicit CAR error, rest are silent failures - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) - second_hgvs = get_hgvs_from_post_mapped(mapped_variants[1].post_mapped) + # All 4 variants share 1 allele. CAR returns an explicit error for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + allele_hgvs = get_hgvs_from_post_mapped(alleles[0].post_mapped) registered_alleles_mock = [ - { - "@id": f"CA{mapped_variants[0].id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], - }, { "errorType": "InvalidHGVS", - "hgvs": second_hgvs, + "hgvs": allele_hgvs, "message": "The HGVS string is invalid.", "description": "error", - "inputLine": second_hgvs, + "inputLine": allele_hgvs, "position": "0", }, ] @@ -1079,31 +1573,19 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # Verify the variant whose HGVS returned an explicit CAR error has error details in annotation_metadata. - # Only 1 annotation should have EXTERNAL_SERVICE_REJECTED since only one CAR error was in the response. - car_rejected_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_service_rejected", + # The 1 shared allele was rejected by ClinGen → one failed event (reason service_rejected). + rejected_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.reason == "service_rejected", ) ).all() - assert len(car_rejected_annotations) == 1 - rejected = car_rejected_annotations[0] - assert rejected.annotation_metadata["submitted_hgvs"] == second_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." - - # The remaining 2 failures (variants 3 and 4) got no CAR response — silent failures get EXTERNAL_API_ERROR. - silent_failure_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_api_error", - ) - ).all() - assert len(silent_failure_annotations) == 2 - for ann in silent_failure_annotations: - assert ann.annotation_metadata["submitted_hgvs"] is not None - assert "car_error_type" not in ann.annotation_metadata + assert len(rejected_events) == 1 + for event in rejected_events: + assert event.disposition == "failed" + assert event.event_metadata["submitted_hgvs"] == allele_hgvs + assert event.event_metadata["car_error_type"] == "InvalidHGVS" + assert event.event_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1185,15 +1667,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1214,19 +1695,18 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( session.refresh(submit_score_set_mappings_to_car_sample_job_run) assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( self, @@ -1255,15 +1735,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1288,19 +1767,18 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_independent( self, @@ -1350,15 +1828,15 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.ERRORED assert submit_score_set_mappings_to_car_sample_job_run.error_message == "ClinGen service error" - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_pipeline( self, @@ -1413,15 +1891,15 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 @pytest.mark.unit @@ -1771,11 +2249,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -1842,11 +2320,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -1949,11 +2427,11 @@ async def dummy_no_linked_alleles_submission(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -1997,7 +2475,7 @@ async def test_submit_score_set_mappings_to_ldh_hgvs_not_found( # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2052,11 +2530,11 @@ async def dummy_submission_failure(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database # TODO:XXX: Change status to 'failed' once decorator supports it @@ -2122,15 +2600,15 @@ async def dummy_partial_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 success_count = 0 failure_count = 0 for ann in annotation_statuses: - if ann.status == "success": + if ann.disposition == "present": success_count += 1 - elif ann.status == "failed": + elif ann.disposition == "failed": failure_count += 1 assert success_count == 1 @@ -2200,11 +2678,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2277,11 +2755,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2349,11 +2827,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2408,7 +2886,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2463,7 +2941,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index a55eb6b88..6d95b917a 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -4,11 +4,14 @@ pytest.importorskip("arq") +from datetime import date from unittest.mock import AsyncMock, patch from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.external_services.clingen_cache import warm_clingen_cache @@ -17,6 +20,51 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: str | None, vrs_digest: str) -> Allele: + """Create a Variant → MappingRecord → Allele chain and return the Allele. + + get_alleles_for_score_set joins: Allele ← MappingRecordAllele (current) ← MappingRecord (current) ← Variant. + Leaving valid_to=NULL on MappingRecord and MappingRecordAllele makes them current. + """ + variant = Variant( + urn=f"urn:variant:{urn_suffix}", + score_set_id=score_set_id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.flush() + + allele = Allele( + vrs_digest=vrs_digest, + level="genomic", + post_mapped={"type": "Allele", "location": {"sequenceReference": {"refgetAccession": vrs_digest}}}, + clingen_allele_id=caid, + ) + session.add(allele) + session.flush() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(mapping_record) + session.flush() + + link = MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + session.add(link) + session.flush() + + return allele + + @pytest.mark.unit @pytest.mark.asyncio class TestWarmClingenCacheUnit: @@ -49,26 +97,15 @@ async def test_warms_cache_for_variants_with_caids( """Job calls get_clingen_allele_data for each distinct allele ID.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) - # Create two variants with the same CAID — should only warm once (distinct) - for i, caid in enumerate(["CA111111", "CA222222", "CA111111"]): - variant = Variant( - urn=f"urn:variant:warm-test-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 1}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 1}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + # Three variants, two sharing the same CAID — should only warm 2 distinct IDs. + # Two separate allele rows are needed (different VRS digests) to get 2 distinct CAIDs. + _make_allele_with_caid(session, score_set.id, "warm-test-0", "CA111111", "digest-warm-0") + _make_allele_with_caid(session, score_set.id, "warm-test-1", "CA222222", "digest-warm-1") + # Third variant points to same allele as first (same CAID CA111111 via a fresh allele row + # with the same digest — but since get_alleles_for_score_set returns per-variant rows and + # caid dedup happens in the warmer, we can share the digest). + _make_allele_with_caid(session, score_set.id, "warm-test-2", "CA111111", "digest-warm-2") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -100,24 +137,8 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): - variant = Variant( - urn=f"urn:variant:warm-filter-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 10}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 10}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-filter-{i}", caid, f"digest-filter-{i}") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -147,24 +168,8 @@ async def test_continues_on_individual_fetch_failure( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) for i, caid in enumerate(["CA444444", "CA555555"]): - variant = Variant( - urn=f"urn:variant:warm-fail-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 20}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 20}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-fail-{i}", caid, f"digest-fail-{i}") + session.commit() # First call raises, second succeeds mock_get_allele_data = AsyncMock( @@ -192,7 +197,7 @@ async def test_only_warms_current_mapped_variants( with_warm_clingen_cache_job, sample_warm_clingen_cache_job_run, ): - """Job only fetches allele IDs from current (not superseded) mapped variants.""" + """Job only fetches allele IDs from current (not superseded) mapping records.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( @@ -203,25 +208,65 @@ async def test_only_warms_current_mapped_variants( data={}, ) session.add(variant) - session.commit() + session.flush() - # Non-current mapped variant should be ignored - old_mv = MappedVariant( - variant_id=variant.id, + # Superseded allele (MappingRecord with valid_to set — not current) + old_allele = Allele( + vrs_digest="digest-old-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA666666", - current=False, - mapped_date="2023-01-01T00:00:00Z", - mapping_api_version="0.9.0", ) - # Current mapped variant should be included - current_mv = MappedVariant( + session.add(old_allele) + session.flush() + + from datetime import datetime, timezone + + old_mapping_record = MappingRecord( variant_id=variant.id, + assay_level="genomic", + mapping_api_version="0.9.0", + mapped_date=date(2023, 1, 1), + ) + # Close this record so it is non-current (valid_to set) + old_mapping_record.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_mapping_record) + session.flush() + + old_link = MappingRecordAllele( + mapping_record_id=old_mapping_record.id, + allele_id=old_allele.id, + is_authoritative=True, + ) + old_link.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_link) + session.flush() + + # Current allele (MappingRecord with valid_to=NULL — current) + current_allele = Allele( + vrs_digest="digest-current-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA777777", - current=True, - mapped_date="2024-01-01T00:00:00Z", + ) + session.add(current_allele) + session.flush() + + current_mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(current_mapping_record) + session.flush() + + current_link = MappingRecordAllele( + mapping_record_id=current_mapping_record.id, + allele_id=current_allele.id, + is_authoritative=True, ) - session.add_all([old_mv, current_mv]) + session.add(current_link) session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 1ac616ffc..f7698d737 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -3,19 +3,20 @@ import pytest import requests -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, FailureCategory, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - pytest.importorskip("arq") from unittest.mock import patch +from sqlalchemy import select + from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -26,6 +27,7 @@ "GeneSymbol": "TEST", "ClinicalSignificance": "benign", "ReviewStatus": "reviewed by expert panel", + "VariationID": "987654", }, } @@ -33,32 +35,29 @@ @pytest.mark.unit @pytest.mark.asyncio class TestRefreshClinvarControlsUnit: - """Tests for the refresh_clinvar_controls job function.""" + """Unit tests for the allele-model refresh_clinvar_controls job.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" + """Pin _generate_clinvar_versions to a single version so clinvar_version == '01_2026'.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_skips_version_on_fetch_failure( + async def test_no_alleles_with_caids( self, mock_worker_ctx, session, + with_populated_domain_data, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): - """Test that a fetch failure for a version is logged and skipped, not propagated.""" - - async def awaitable_exception(*args, **kwargs): - raise Exception("Network error") - + """No current alleles carry a CAID -> the job succeeds with nothing to do.""" with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=awaitable_exception, + return_value=MOCK_CLINVAR_DATA, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -68,20 +67,26 @@ async def awaitable_exception(*args, **kwargs): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarControl)).all() == [] + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] - async def test_refresh_clinvar_controls_no_mapped_variants( + async def test_fetch_failure_skips_version( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when there are no mapped variants.""" + """A version whose TSV fetch fails is logged and skipped, not propagated.""" + + async def boom(*args, **kwargs): + raise Exception("Network error") with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value={}, + side_effect=boom, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -89,35 +94,21 @@ async def test_refresh_clinvar_controls_no_mapped_variants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - async def test_refresh_clinvar_controls_no_variants_have_caids( + async def test_multi_variant_caid_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.2G>A", - hgvs_pro="NP_000000.1:p.Val2Ile", - data={"hgvs_c": "NM_000000.1:c.2G>A", "hgvs_p": "NP_000000.1:p.Val2Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) + """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() with patch( @@ -130,34 +121,40 @@ async def test_refresh_clinvar_controls_no_variants_have_caids( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." + # Allele-keyed event: a cis-block CAID structurally can't key ClinVar — a pipeline gap, not a + # statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.MULTI_VARIANT_CAID + assert event.allele_id == allele.id and event.variant_id is None - async def test_refresh_clinvar_controls_variants_are_multivariants( + async def test_protein_level_allele_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when all variants are multi-variant CAIDs.""" - # Update the mapped variant to have a multi-variant CAID - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" + """A protein-level allele is never linked to ClinVar (a nucleotide-level DB): its clinical + calls come from its g./c. siblings. Skipped as not-applicable before any resolution runs.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" session.commit() - with patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + with ( + patch( + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", + ) as resolve_spy, + patch( + "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", + return_value=MOCK_CLINVAR_DATA, + ), ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -165,37 +162,35 @@ async def test_refresh_clinvar_controls_variants_are_multivariants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) - - async def test_refresh_clinvar_controls_clingen_api_failure( + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(ClinvarControl)).all() == [] + # Short-circuited before touching ClinGen — a protein allele can never resolve to a ClinVar id. + resolve_spy.assert_not_awaited() + + # Allele-keyed not-applicable event: ClinVar is nucleotide-level, so a protein allele is a + # structural gap (like a multi-variant CAID), not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles ClinGen API failures gracefully.""" + """ClinGen returns no ClinVar allele id for the CAID -> absent/no_record event, no link.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to raise an exception with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), + return_value=None, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -208,40 +203,33 @@ async def test_refresh_clinvar_controls_clingen_api_failure( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - # Verify an annotation status was created for the variant due to ClinGen API failure - mapped_variant = session.query(MappedVariant).first() - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + # ClinGen had no ClinVar AlleleID for this CAID — an informative negative about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None + + async def test_clinvar_data_not_found_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no associated ClinVar Allele ID gracefully.""" + """The resolved ClinVar allele id is absent from the version's TSV -> absent/no_record, no link.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return None with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + return_value="VCV000000123", ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value={"VCV000000999": {"GeneSymbol": "X", "ClinicalSignificance": "y", "ReviewStatus": "z"}}, ), ): result = await refresh_clinvar_controls( @@ -250,48 +238,34 @@ async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message - - async def test_refresh_clinvar_controls_no_clinvar_data_found( + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + # The CAID resolved to a ClinVar AlleleID, but it's absent from this release's snapshot — a + # genuine, version-scoped negative. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None + + async def test_clingen_api_failure_fails_when_total( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no ClinVar data found for the associated ClinVar Allele ID.""" - - # TSV data with a different allele ID than the one being looked up - non_matching_clinvar_data = { - "VCV000000001": { - "GeneSymbol": "TEST", - "ClinicalSignificance": "benign", - "ReviewStatus": "reviewed by expert panel", - }, - } + """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=requests.exceptions.RequestException("ClinGen API error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=non_matching_clinvar_data, + return_value=MOCK_CLINVAR_DATA, ), ): result = await refresh_clinvar_controls( @@ -300,31 +274,24 @@ async def test_refresh_clinvar_controls_no_clinvar_data_found( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message + assert result.status == JobStatus.FAILED + assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.allele_id == allele.id and event.variant_id is None - async def test_refresh_clinvar_controls_successful_annotation_existing_control( + async def test_successful_link_new_control( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data.""" + """A resolved + present CAID creates a ClinvarControl, a live link, and a present/created event.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -341,54 +308,53 @@ async def test_refresh_clinvar_controls_successful_annotation_existing_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - mapped_variant = session.query(MappedVariant).first() - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_successful_annotation_new_control( + assert result.data["created_link_count"] == 1 + + control = session.scalars(select(ClinvarControl)).one() + assert control.db_identifier == "VCV000000123" + assert control.db_version == "01_2026" + assert control.clinical_significance == "benign" + # AlleleID stays in db_identifier; the canonical VariationID is captured alongside it. + assert control.clinvar_variation_id == "987654" + + links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert len(links) == 1 + assert links[0].clinvar_control_id == control.id + + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.PRESENT + assert event.reason == EventReason.CREATED + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.allele_id == allele.id and event.variant_id is None + + async def test_links_and_annotates_full_allele_set( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_rt_derived_allele_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant and mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.3C>T", - hgvs_pro="NP_000000.1:p.Ala3Val", - data={"hgvs_c": "NM_000000.1:c.3C>T", "hgvs_p": "NP_000000.1:p.Ala3Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA124", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + """ClinVar linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the matching CAID and is linked. Events are now allele-keyed, so the RT-derived + allele's present status is recorded directly — the limitation the per-variant bandaid had + (dropping the RT-derived allele's status) is lifted. Each allele gets its own event: present for + the linked RT allele, absent for the unmatched authoritative one. + """ + _, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + rt_caid = rt_allele.clingen_allele_id + + async def resolve(caid): + # Only the RT-derived allele's CAID resolves to a ClinVar id present in the TSV. + return "VCV000000123" if caid == rt_caid else None - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=resolve, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -401,1105 +367,293 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_idempotent_run( + # The RT-derived allele IS linked — the core fix. + rt_links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == rt_allele.id, ClinvarAlleleLink.current) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no ClinVar match, so it is not linked. + assert ( + session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == authoritative_allele.id) + ).all() + == [] + ) + + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, the + # unmatched authoritative allele is absent. The variant resolves its derived allele's status + # through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == AnnotationType.CLINVAR_CONTROL for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == Disposition.PRESENT + assert by_allele[rt_allele.id].reason == EventReason.CREATED + assert by_allele[authoritative_allele.id].disposition == Disposition.ABSENT + assert by_allele[authoritative_allele.id].reason == EventReason.NO_RECORD + + async def test_idempotent_rerun_skips_and_does_not_duplicate( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that running the job multiple times does not create duplicate annotation statuses.""" + """A second run finds the allele already linked at the version, skips the resolution, reports + preexisting, and creates neither a duplicate control nor a duplicate link.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], + return_value=MOCK_CLINVAR_DATA, ), ): - # First run - result1 = await refresh_clinvar_controls( + first = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - session.commit() + assert resolve_spy.await_count == 1 - # Second run - result2 = await refresh_clinvar_controls( + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - async def test_refresh_clinvar_controls_partial_failure( + # Second run skipped the resolution entirely (version-keyed skip). + assert resolve_spy.await_count == 1 + + assert first.data["created_link_count"] == 1 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + + # One control, one live link — no churn. + assert len(session.scalars(select(ClinvarControl)).all()) == 1 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Per-version events: two allele-keyed events (one per run), no current flag. The first run + # created the link; the second found it preexisting. Latest event (by id) wins. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id).order_by(AnnotationEvent.id) + ).all() + assert len(events) == 2 + assert events[0].reason == EventReason.CREATED + assert events[1].reason == EventReason.PREEXISTING + assert all(e.disposition == Disposition.PRESENT for e in events) + + async def test_release_reresolution_supersedes_newest_wins( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.4G>C", - hgvs_pro="NP_000000.1:p.Gly4Ala", - data={"hgvs_c": "NM_000000.1:c.4G>C", "hgvs_p": "NP_000000.1:p.Gly4Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA125", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) + """Defensive guard: if an allele re-resolves to a *different* control within the same release + (should never happen — archival data is immutable), the old live link is superseded + newest-wins, leaving exactly one live link per (allele, version).""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA125": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" + tsv = { + "VCV000000123": { + "GeneSymbol": "A", + "ClinicalSignificance": "benign", + "ReviewStatus": "ok", + "VariationID": "111", + }, + "VCV000000999": { + "GeneSymbol": "B", + "ClinicalSignificance": "pathogenic", + "ReviewStatus": "ok", + "VariationID": "222", + }, + } with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, + side_effect=["VCV000000123", "VCV000000999"], ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value=tsv, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - async def test_total_api_failure_returns_failed( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that the job returns FAILED when all ClinVar lookups fail.""" - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( + session.commit() + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - async def test_upsert_does_not_create_duplicate_control_when_row_already_exists( + control_a = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000123")).one() + control_b = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000999")).one() + + all_links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + live_links = [link for link in all_links if link.valid_to is None] + + # Exactly one live link per (allele, version) — the new control — and the old one is retired. + assert len(live_links) == 1 + assert live_links[0].clinvar_control_id == control_b.id + retired = [link for link in all_links if link.clinvar_control_id == control_a.id] + assert len(retired) == 1 + assert retired[0].valid_to is not None + # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. + assert retired[0].valid_to == live_links[0].valid_from + # Stamped from the DB clock (func.now()), so the boundary is timezone-aware and comparable to + # every other func.now()-stamped valid-time row — a regression to a naive datetime.now() would + # land here as a tz-naive value. + assert live_links[0].valid_from.tzinfo is not None + assert retired[0].valid_to.tzinfo is not None + + async def test_force_reresolves_without_duplicating_link( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the upsert handles a pre-existing ClinicalControl row without creating a duplicate. - - This covers the concurrent-job race condition where two refresh_clinvar_controls - jobs run simultaneously for different score sets and both try to insert a - ClinicalControl for the same (db_name, db_identifier, db_version). The second - job's upsert must hit ON CONFLICT DO UPDATE rather than inserting a second row. - """ - # Simulate the state left by a concurrent job: the ClinicalControl row - # for this identifier/version already exists in the DB with stale data. - pre_existing_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - db_version="01_2026", - gene_symbol="OLD_SYMBOL", - clinical_significance="likely pathogenic", - clinical_review_status="criteria provided, single submitter", - ) - session.add(pre_existing_control) + """force bypasses the version-keyed skip and re-resolves, but the get-or-create link write + does not duplicate an existing live link.""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + session.commit() + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.SUCCEEDED - - # Only one row should exist — the upsert must not have inserted a second one. - controls = session.query(ClinicalControl).filter_by(db_identifier="VCV000000123", db_version="01_2026").all() - assert len(controls) == 1 - - # The upsert should have updated the stale data from the concurrent job. - session.refresh(controls[0]) - assert controls[0].gene_symbol == "TEST" - assert controls[0].clinical_significance == "benign" - assert controls[0].clinical_review_status == "reviewed by expert panel" + # force re-resolved on the second run despite an existing link. + assert resolve_spy.await_count == 2 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None @pytest.mark.integration @pytest.mark.asyncio -class TestRefreshClinvarControlsIntegration: - """Integration tests for the refresh_clinvar_controls job function.""" +class TestRefreshClinvarControlsArqContext: + """End-to-end tests for refresh_clinvar_controls within an ARQ worker context.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when there are no mapped variants.""" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify no controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_variants_with_caid( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.5T>A", - hgvs_pro="NP_000000.1:p.Leu5Gln", - data={"hgvs_c": "NM_000000.1:c.5T>A", "hgvs_p": "NP_000000.1:p.Leu5Gln"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controlsvariants_are_multivariants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when all variants are multi-variant CAIDs.""" - # Add a variant with a multi-variant CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-multicid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.6A>G", - hgvs_pro="NP_000000.1:p.Thr6Ala", - data={"hgvs_c": "NM_000000.1:c.6A>G", "hgvs_p": "NP_000000.1:p.Thr6Ala"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA-MULTI-003,CA-MULTI-004", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_arq_context_successful_link( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job handles no associated ClinVar Allele ID gracefully.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.7C>A", - hgvs_pro="NP_000000.1:p.Ser7Tyr", - data={"hgvs_c": "NM_000000.1:c.7C>A", "hgvs_p": "NP_000000.1:p.Ser7Tyr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA126", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return None + """The job links an allele and records a present event under an ARQ worker.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + return_value="VCV000000123", ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message + assert len(session.scalars(select(ClinvarControl)).all()) >= 1 + assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == Disposition.PRESENT + assert events[0].annotation_type == AnnotationType.CLINVAR_CONTROL + assert events[0].allele_id is not None and events[0].variant_id is None - # Verify job run status is marked as completed session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - async def test_refresh_clinvar_controls_no_clinvar_data( + async def test_arq_context_exception_handling( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job handles no ClinVar data found for the associated ClinVar Allele ID.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.8G>T", - hgvs_pro="NP_000000.1:p.Val8Phe", - data={"hgvs_c": "NM_000000.1:c.8G>T", "hgvs_p": "NP_000000.1:p.Val8Phe"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA127", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID + """An unexpected error during resolution is caught by the decorators and the job errors.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000001", + side_effect=ValueError("Unexpected error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_slack, ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + mock_slack.assert_called_once() + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_existing_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.9A>C", - hgvs_pro="NP_000000.1:p.Lys9Thr", - data={"hgvs_c": "NM_000000.1:c.9A>C", "hgvs_p": "NP_000000.1:p.Lys9Thr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA128", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - clinical_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - clinical_significance="likely pathogenic", - gene_symbol="TEST", - clinical_review_status="criteria provided, single submitter", - db_version="01_2026", - ) - session.add(clinical_control) - session.commit() - - mapped_variant.clinical_controls.append(clinical_control) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was updated - session.refresh(clinical_control) - assert clinical_control.clinical_significance == "benign" - assert clinical_control.clinical_review_status == "reviewed by expert panel" - assert mapped_variant in clinical_control.mapped_variants - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_new_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.10C>G", - hgvs_pro="NP_000000.1:p.Pro10Arg", - data={"hgvs_c": "NM_000000.1:c.10C>G", "hgvs_p": "NP_000000.1:p.Pro10Arg"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA129", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_pipeline_context( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_pipeline, - sample_refresh_clinvar_controls_job_in_pipeline, - ): - """Integration test: job successfully annotates a variant with ClinVar control data in a pipeline context.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_in_pipeline.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.12G>A", - hgvs_pro="NP_000000.1:p.Met12Ile", - data={"hgvs_c": "NM_000000.1:c.12G>A", "hgvs_p": "NP_000000.1:p.Met12Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_in_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_in_pipeline) - assert sample_refresh_clinvar_controls_job_in_pipeline.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - session.refresh(sample_refresh_clinvar_controls_pipeline) - assert sample_refresh_clinvar_controls_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_idempotent_run( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: running the job multiple times does not create duplicate annotation statuses.""" - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], - ), - ): - # First run - result1 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - session.commit() - # reset the job run status to pending for the second run - sample_refresh_clinvar_controls_job_run.status = JobStatus.PENDING - session.commit() - - # Second run - result2 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_partial_failure( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:integration-test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.11G>C", - hgvs_pro="NP_000000.1:p.Gly11Ala", - data={"hgvs_c": "NM_000000.1:c.11G>C", "hgvs_p": "NP_000000.1:p.Gly11Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA130": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - # Verify a clinical control was added for the successfully annotated variant and not the unsuccessful one - clinical_control1 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant1)).one() - ) - assert clinical_control1.db_identifier == "VCV000000123" - - clinical_control2 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant2)).all() - ) - assert len(clinical_control2) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_propagates_exceptions_to_decorator( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that unexpected exceptions are propagated.""" - - # Mock the get_associated_clinvar_allele_id function to raise an unexpected exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestRefreshClinvarControlsArqContext: - """Tests for running the refresh_clinvar_controls job function within an ARQ worker context.""" - - @pytest.fixture(autouse=True) - def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" - with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", - return_value=[(2026, 1)], - ): - yield - - async def test_refresh_clinvar_controls_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context in a pipeline context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - pass - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context in a pipeline context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - # Verify the pipeline is marked as failed - pass diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index fc8e211c0..03a3cefac 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -4,13 +4,20 @@ pytest.importorskip("arq") -from unittest.mock import MagicMock, patch +from unittest.mock import patch +from sqlalchemy import select + +from mavedb.lib import gnomad as gnomad_lib +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -22,7 +29,7 @@ class TestLinkGnomadVariantsUnit: """Unit tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -30,7 +37,7 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test linking gnomAD variants when no mapped variants have CAIDs.""" + """No authoritative alleles with CAIDs -> the job succeeds with nothing to do.""" result = await link_gnomad_variants( mock_worker_ctx, 1, @@ -47,7 +54,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test linking gnomAD variants when no gnomAD variants match the CAIDs.""" @@ -55,7 +62,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value={}, + return_value=[], ), patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -68,6 +75,44 @@ async def test_link_gnomad_variants_no_gnomad_matches( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_protein_level_allele_skipped( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to gnomAD (a nucleotide-level, genomic-coordinate + resource): its frequencies come from its g./c. siblings. Skipped as not-applicable, and it + never reaches the Athena query — sparing the round-trip.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + ) as fetch_spy: + result = await link_gnomad_variants( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_link_gnomad_variants_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(GnomadAlleleLink)).all() == [] + # The protein allele's CAID was dropped from the query set, so Athena is never queried. + fetch_spy.assert_not_called() + + # Allele-keyed not-applicable event: gnomAD is nucleotide-level, so a protein allele is a + # structural gap, not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.GNOMAD_ALLELE_FREQUENCY + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_link_gnomad_variants_call_linking_method( self, session, @@ -75,7 +120,7 @@ async def test_link_gnomad_variants_call_linking_method( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that the linking method is called when gnomAD variants match CAIDs.""" @@ -83,11 +128,11 @@ async def test_link_gnomad_variants_call_linking_method( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value=[MagicMock()], + return_value=[object()], ), patch( - "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_mapped_variants", - return_value=1, + "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", + return_value={}, ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -108,7 +153,7 @@ async def test_link_gnomad_variants_propagates_exceptions( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are propagated.""" @@ -134,7 +179,7 @@ async def test_link_gnomad_variants_propagates_exceptions( class TestLinkGnomadVariantsIntegration: """Integration tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -142,19 +187,18 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test the end-to-end functionality of the link_gnomad_variants job when no variants have CAIDs.""" + """Test the end-to-end functionality of the link_gnomad_variants job when no alleles have CAIDs.""" result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify no annotations were rendered (since there were no variants with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + # Verify no annotations were rendered (since there were no alleles with CAIDs) + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -167,13 +211,13 @@ async def test_link_gnomad_variants_no_matching_caids( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job when no matching CAIDs are found.""" - # Update the created mapped variant to have a CAID that won't match any gnomAD data - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "NON_MATCHING_CAID" + # Update the allele to have a CAID that won't match any seeded gnomAD data + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "NON_MATCHING_CAID" session.commit() # Patch the athena engine to use the mock athena_engine fixture @@ -183,15 +227,16 @@ async def test_link_gnomad_variants_no_matching_caids( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there were variants with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # Verify an absent event was rendered: the allele's CAID was queried but gnomAD had no record. + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id == allele.id and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -204,10 +249,11 @@ async def test_link_gnomad_variants_successful_linking_independent( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking.""" + _, allele = setup_sample_alleles_with_caid # Patch the athena engine to use the mock athena_engine fixture with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -216,20 +262,173 @@ async def test_link_gnomad_variants_successful_linking_independent( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that a gnomAD variant was created and a live link to the allele established + assert len(session.scalars(select(GnomADVariant)).all()) > 0 + live_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(live_links) == 1 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_links_and_annotates_rt_derived_allele( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_rt_derived_allele_with_caid, + athena_engine, + ): + """gnomAD linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the CAID gnomAD matches and is linked. Events are now allele-keyed, so the + RT-derived allele's PRESENT status is recorded directly — the limitation the per-variant + bandaid had (dropping the RT-derived allele's status) is lifted. Each allele in the score set + gets its own event: present for the linked RT allele, absent for the unmatched authoritative + one.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + assert result.status == JobStatus.SUCCEEDED + + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == rt_allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no gnomAD match, so it gets no link. + assert ( + len( + session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == authoritative_allele.id) + ).all() + ) + == 0 + ) + + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, + # the unmatched authoritative allele is absent. The variant resolves its derived allele's + # status through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "gnomad_allele_frequency" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" + + async def test_link_gnomad_variants_skips_allele_already_current( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """An allele already linked at the current gnomAD version is skipped: no Athena query, the + status reports SUCCESS/preexisting, and the existing link is not churned.""" + _, allele = setup_sample_alleles_with_caid + + # Simulate a prior run: a live link at the current gnomAD version. + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + with patch("mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids") as fetch_spy: + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + # Link not churned: still exactly one, still live. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Event is PRESENT, marked preexisting (the link was already current; not created this run). + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" + + async def test_link_gnomad_variants_force_refetches_without_churn( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + athena_engine, + ): + """force bypasses the skip and re-fetches, but the linker supersedes only on change, so a + forced re-run of unchanged data reports preexisting and does not churn the link.""" + _, allele = setup_sample_alleles_with_caid + + # Prior live link pointing at the same variant the Athena mock resolves to (1-12345-G-A). + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=23, + allele_number=32432423, + allele_frequency=23 / 32432423, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + sample_link_gnomad_variants_run.job_params = {**sample_link_gnomad_variants_run.job_params, "force": True} + session.commit() + + with ( + patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), + patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + side_effect=gnomad_lib.gnomad_variant_data_for_caids, + ) as fetch_spy, + ): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_called_once() # force bypassed the version-keyed skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + # Unchanged → no churn. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + async def test_link_gnomad_variants_successful_linking_pipeline( self, session, @@ -237,7 +436,7 @@ async def test_link_gnomad_variants_successful_linking_pipeline( mock_worker_ctx, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking in a pipeline.""" @@ -249,15 +448,16 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -274,7 +474,7 @@ async def test_link_gnomad_variants_exceptions_handled_by_decorators( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are handled by decorators.""" @@ -317,7 +517,7 @@ async def test_link_gnomad_variants_with_arq_context_independent( with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture.""" @@ -328,15 +528,16 @@ async def test_link_gnomad_variants_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run) @@ -351,7 +552,7 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( athena_engine, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture in a pipeline.""" @@ -362,15 +563,16 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -389,7 +591,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -406,13 +608,12 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run) @@ -427,7 +628,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline athena_engine, sample_link_gnomad_variants_pipeline, sample_link_gnomad_variants_run_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -444,13 +645,12 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run_pipeline) diff --git a/tests/worker/jobs/external_services/test_hgvs.py b/tests/worker/jobs/external_services/test_hgvs.py deleted file mode 100644 index 946724cc5..000000000 --- a/tests/worker/jobs/external_services/test_hgvs.py +++ /dev/null @@ -1,544 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.hgvs import populate_hgvs_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - -SAMPLE_CA_ALLELE_DATA = { - "genomicAlleles": [ - { - "referenceGenome": "GRCh38", - "hgvs": ["NC_000001.11:g.12345A>G"], - } - ], - "transcriptAlleles": [ - { - "hgvs": ["NM_000000.1:c.1A>G"], - "proteinEffect": {"hgvs": "NP_000000.1:p.Met1Val"}, - "MANE": { - "nucleotide": {"RefSeq": {"hgvs": "NM_000000.1:c.1A>G"}}, - "protein": {"RefSeq": {"hgvs": "NP_000000.1:p.Met1Val"}}, - }, - } - ], -} - -SAMPLE_PA_ALLELE_DATA = { - "aminoAcidAlleles": [ - { - "hgvs": ["NP_000000.1:p.Met1Val"], - } - ], -} - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateHgvsForScoreSetUnit: - """Unit tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test populating HGVS when no mapped variants exist.""" - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - async def test_variant_without_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant without a CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_variant_with_multi_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant with a multi-variant CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = "CA123,CA456" - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_successful_ca_allele_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test successful HGVS population for a CA allele.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["populated_count"] == 1 - - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - session.refresh(mapped_variant) - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - async def test_clingen_api_error_recorded_as_failed( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that ClinGen API errors are recorded as failed annotations.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - - async def test_total_api_failure_sends_slack_alert( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a Slack alert is sent when all variants fail HGVS population.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - patch("mavedb.worker.jobs.external_services.hgvs.log_and_send_slack_message") as mock_slack, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - assert result.data["populated_count"] == 0 - mock_slack.assert_called_once() - - async def test_clingen_allele_not_found_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a 404 from ClinGen results in a skipped annotation.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=None, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetIntegration: - """Integration tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end successful HGVS population.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated with HGVS - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population_pipeline( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end HGVS population in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that variants without CAIDs get a skipped annotation status.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - sample_populate_hgvs_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetArqContext: - """Tests for populate_hgvs_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job completed - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_pipeline, - sample_populate_hgvs_run_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.ERRORED - - # Verify pipeline failed - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_variant_translation.py b/tests/worker/jobs/external_services/test_variant_translation.py deleted file mode 100644 index 3e1f364ea..000000000 --- a/tests/worker/jobs/external_services/test_variant_translation.py +++ /dev/null @@ -1,770 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from sqlalchemy import select - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.models.variant_translation import VariantTranslation -from mavedb.worker.jobs.external_services.variant_translation import populate_variant_translations_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -# --- Unit Tests --- - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateVariantTranslationsUnit: - """Unit tests for the populate_variant_translations_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test that the job succeeds with zero translations when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_variant_without_caid_no_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a variant without a CAID results in no translations.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_ca_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele creates translations via PA lookup.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00001"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA11111", "CA22222"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # 1 for PA00001->CA9765210 (the original CA), 2 for PA00001->CA11111 and PA00001->CA22222 - assert result.data["translations_created"] == 3 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 3 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation is not None - - async def test_pa_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele creates translations via CA lookup.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA99999" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA33333", "CA44444"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 2 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 - aa_ids = {t.aa_clingen_id for t in translations} - assert aa_ids == {"PA99999"} - - async def test_multi_variant_caid_expanded( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that comma-separated CAIDs are expanded and each processed independently.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "CA55555,CA66666" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00002"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # PA00002->CA55555 and PA00002->CA66666 - assert result.data["translations_created"] == 2 - - async def test_ca_allele_no_pa_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele with no canonical PA IDs results in a skip.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "skipped" - - async def test_pa_allele_no_ca_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele with no registered CA IDs results in a skip.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA88888" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - async def test_ca_allele_api_failure_records_failed_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a ClinGen API failure for CA allele records a failed annotation.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "failed" - - async def test_unrecognized_allele_format_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that an unrecognized allele ID format is skipped.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - - async def test_duplicate_translations_not_created( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that duplicate translations are not created on re-run.""" - # Pre-populate a translation - session.add(VariantTranslation(aa_clingen_id="PA00003", nt_clingen_id="CA9765210")) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00003"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - translations = session.scalars( - select(VariantTranslation).where(VariantTranslation.aa_clingen_id == "PA00003") - ).all() - assert len(translations) == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - async def test_multiple_alleles_sharing_pa_no_duplicate_error( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that two CA alleles mapping to the same PA don't cause a UniqueViolation. - - This is a regression test for a bug where the SELECT-then-INSERT upsert pattern - failed to detect in-session duplicates: both alleles' iterations called - upsert_variant_translations with overlapping (PA, CA) pairs, the SELECT found - no committed row, both staged db.add() for the same pair, and the subsequent - update_progress commit raised a UniqueViolation. - """ - # Add a second variant with a different CA allele under the same score set. - score_set_id = sample_populate_variant_translations_run.job_params["score_set_id"] - - variant2 = Variant( - urn="urn:variant:test-second-ca-allele", - score_set_id=score_set_id, - hgvs_nt="NM_000000.1:c.2T>G", - hgvs_pro="NP_000000.1:p.Val2Gly", - data={}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA_SECOND", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Both CA alleles resolve to the same PA. The PA then returns the same set of - # registered CAs for both iterations, producing fully overlapping translation pairs. - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA_SHARED"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA9765210", "CA_SECOND"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - pairs = {(t.aa_clingen_id, t.nt_clingen_id) for t in translations} - # PA_SHARED paired with each CA: CA9765210 (original from allele 1), - # CA_SECOND (original from allele 2), plus both as registered CAs. - assert ("PA_SHARED", "CA9765210") in pairs - assert ("PA_SHARED", "CA_SECOND") in pairs - - async def test_total_api_failure_returns_failed( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job returns FAILED when all variant translation lookups fail.""" - import requests - - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - assert result.data["translations_created"] == 0 - - -# --- Integration Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsIntegration: - """Integration tests that exercise the full decorator stack.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_successful_job_updates_status( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a successful job run updates the job status to SUCCEEDED.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00004"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA77777"], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00004->CA9765210 and PA00004->CA77777 - - async def test_job_with_pipeline_updates_pipeline_status( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a job in a pipeline updates the pipeline status on success.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00005"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline.id, - ) - - session.refresh(sample_populate_variant_translations_run_pipeline) - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that variants without CAIDs produce no annotations (filtered before processing).""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_unrecognized_allele_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unrecognized allele formats create skipped annotations through the full stack.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - -# --- ARQ Context Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsArqContext: - """Tests for populate_variant_translations_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00006"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA88888"], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00006->CA9765210 and PA00006->CA88888 - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00007"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_pipeline, - sample_populate_variant_translations_run_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.ERRORED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 611c33894..19587f5c2 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -4,29 +4,49 @@ pytest.importorskip("arq") +from datetime import date, timedelta from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, JobStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.lib.vep import VepResolution from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +_RESOLVE = "mavedb.worker.jobs.external_services.vep._resolve_consequences" +_RELEASE = "mavedb.worker.jobs.external_services.vep.get_ensembl_release" +_ENSEMBL_RELEASE = "116" + + +@pytest.fixture(autouse=True) +def mock_ensembl_release(): + """Stamp every job run with a fixed Ensembl release so tests version-key deterministically without + hitting /info/software. Tests exercising a release-fetch failure override this with an inner patch.""" + with patch(_RELEASE, return_value=_ENSEMBL_RELEASE): + yield + + +def _live_consequences_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + @pytest.mark.asyncio @pytest.mark.unit class TestPopulateVepForScoreSetUnit: - """Unit tests for populate_vep_for_score_set.""" + """Unit tests for the populate_vep_for_score_set job.""" - async def test_no_mapped_variants( + async def test_no_alleles_with_hgvs( self, session, with_populated_domain_data, @@ -34,7 +54,7 @@ async def test_no_mapped_variants( mock_worker_ctx, sample_populate_vep_run, ): - """Job succeeds with zero counts when no mapped variants exist.""" + """No alleles with HGVS -> the job succeeds with nothing to do.""" result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -43,405 +63,451 @@ async def test_no_mapped_variants( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - assert result.data["variants_with_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["created_allele_count"] == 0 + assert result.data["preexisting_allele_count"] == 0 + assert result.data["absent_allele_count"] == 0 + assert result.data["errored_allele_count"] == 0 - async def test_variant_without_hgvs_assay_level_skipped( + async def test_calls_resolver_when_alleles_present( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A mapped variant with no hgvs_assay_level gets a SKIPPED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - mapped_variant.hgvs_assay_level = None - session.commit() - - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" + with patch(_RESOLVE, return_value=VepResolution({}, set())) as mock_resolve: + result = await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SKIPPED - assert annotation.failure_category == AnnotationFailureCategory.MISSING_IDENTIFIER + mock_resolve.assert_called_once() - async def test_vep_api_success_sets_consequence_and_annotation( + async def test_propagates_exceptions( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """VEP returns a consequence: mapped variant and SUCCESS annotation are updated.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 1 - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS - - async def test_vep_missing_triggers_variant_recoder_fallback( + """Exceptions raised while resolving consequences are propagated.""" + with patch(_RESOLVE, side_effect=Exception("Test exception")): + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "Test exception" + + async def test_aborts_when_release_unavailable( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When VEP misses a variant, Variant Recoder is called and its result fed back to VEP.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" - + """If the Ensembl release cannot be fetched the job aborts rather than mis-versioning its writes + — the version is load-bearing for the skip, so a failure must propagate, not be swallowed.""" with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # initial VEP pass returns nothing - {genomic_hgvs: "missense_variant"}, # second VEP pass on recoded HGVS - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), + patch(_RELEASE, side_effect=Exception("info/software unavailable")), + patch(_RESOLVE) as mock_resolve, ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + assert str(exc_info.value) == "info/software unavailable" + mock_resolve.assert_not_called() # never queried VEP without a version to stamp + + +@pytest.mark.asyncio +@pytest.mark.integration +class TestPopulateVepForScoreSetIntegration: + """Integration tests for the populate_vep_for_score_set job.""" + + async def test_no_alleles_with_hgvs( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + ): + """End-to-end: no alleles -> no consequence rows, no annotations, job succeeds.""" + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED - async def test_variant_recoder_failure_annotated_as_failed( + async def test_no_consequence_resolved( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Variant Recoder returning no result for an HGVS produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep + """A genuine empty (VEP queried successfully, found nothing) writes no consequence row and an + ABSENT/no_record event — a trustworthy negative, now that empty is split from a failed request.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={}, # recoder has no result - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value=VepResolution({}, set())): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 1 + assert result.data["absent_allele_count"] == 1 + assert len(_live_consequences_for(session, allele.id)) == 0 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None - async def test_vep_failure_after_recoder_annotated_as_failed( + async def test_request_failure_recorded_as_errored( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """VEP returning no consequence even after Variant Recoder produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """An allele whose VEP/Recoder request *failed* is FAILED/api_error — distinct from a genuine + empty — so a transient outage is never mistaken for a confirmed absence.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, # VEP returns nothing in both passes - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + # Resolver reports the allele's HGVS as errored (request failed), not as a hit or an empty. + with patch(_RESOLVE, return_value=VepResolution({}, {allele.hgvs_c})): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["errored_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "failed" + assert events[0].reason == "api_error" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_successful_linking_independent( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" + _, allele = setup_sample_alleles_for_vep - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - async def test_vep_none_consequence_routes_to_recoder( + assert result.status == JobStatus.SUCCEEDED + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE + assert live[0].access_date == date.today() + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_rt_derived_allele_for_vep, ): - """A None consequence from VEP Phase 1 is treated as a miss and routed through Recoder. + """VEP linkage covers the full allele set, not just authoritative links: the RT-derived allele's + genomic HGVS resolves and gets a consequence row. Events are allele-keyed, so each allele in the + score set gets its own event — present for the resolved RT allele, absent/no_record for the + authoritative allele VEP queried but found nothing (a genuine empty, distinct from a request + failure). The per-variant bandaid's limitation (dropping the RT-derived allele's status) is lifted.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep + + # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding + # HGVS is queried successfully but yields no consequence this run (a genuine empty, not errored). + with patch(_RESOLVE, return_value=VepResolution({rt_allele.hgvs_g: "missense_variant"}, set())): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - VEP can return an entry with most_severe_consequence=None when it recognises a variant - but cannot classify it. This should not silently fall into the UNKNOWN outcome branch — - instead it should be treated identically to an absent entry and sent to Recoder. - """ - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + assert result.status == JobStatus.SUCCEEDED - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {hgvs: None}, # VEP knows the variant but returns no consequence - {genomic_hgvs: "missense_variant"}, # Phase 3 on recoded genomic string - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_live = _live_consequences_for(session, rt_allele.id) + assert len(rt_live) == 1 + assert rt_live[0].functional_consequence == "missense_variant" + # The authoritative allele's HGVS had no consequence, so it gets no row. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 + + # One allele-keyed event per allele (never per-variant): the resolved RT allele is present, the + # unclassifiable authoritative allele is absent/no_record (queried, genuinely empty). + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "vep_functional_consequence" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" + + async def test_skips_allele_already_at_current_release( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele with a live consequence already at the current Ensembl release is skipped: no VEP + query, the status reports SUCCESS/preexisting, and the existing row is not churned.""" + _, allele = setup_sample_alleles_for_vep + + # Simulate a prior run at the current release. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today(), ) + ) + session.commit() - assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + mock_resolve.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - async def test_most_severe_consequence_selected_from_multiple_genomic_hgvs( + # Row not churned: still exactly one, still live. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" + + async def test_new_release_same_consequence_bumps_in_place( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When Recoder returns multiple genomic strings with different consequences, the most severe wins.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_less_severe = "NC_000017.11:g.43094692C>T" # → missense_variant - genomic_more_severe = "NC_000017.11:g.43094692C>A" # → stop_gained - - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # Phase 1: VEP misses - { - genomic_less_severe: "missense_variant", - genomic_more_severe: "stop_gained", - }, # Phase 3: two different consequences - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_less_severe, genomic_more_severe]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; an unchanged consequence advances + source_version (and access_date) in place — no supersede, so a new release does not churn + history for a categorical value that did not change.""" + _, allele = setup_sample_alleles_for_vep + + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_with_consequences"] == 1 + mock_resolve.assert_called_once() # older release -> not skipped, re-queried + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "stop_gained" + # Unchanged value -> no supersede: one row, still live, version + date advanced in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].source_version == _ENSEMBL_RELEASE + assert rows[0].access_date == date.today() - async def test_multiple_variants_sharing_hgvs_all_get_consequence( + async def test_force_requeries_unchanged_without_churn( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """All mapped variants that share an HGVS string each receive the consequence.""" - _, mapped_variant_1 = setup_sample_variants_for_vep - hgvs = mapped_variant_1.hgvs_assay_level - - score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) - variant_2 = Variant( - urn="urn:variant:test-variant-for-vep-2", - score_set_id=score_set.id, - hgvs_nt=hgvs, - data={"hgvs_c": hgvs}, + """force bypasses the current-release skip and re-queries, but the linker only supersedes on a + value change: a forced re-run that resolves the same consequence reports preexisting and does + not churn the row (version/access_date advanced in place).""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence already at the current release (would be skipped without force). + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today() - timedelta(days=1), + ) ) - session.add(variant_2) session.commit() - mapped_variant_2 = MappedVariant( - variant_id=variant_2.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": hgvs, "syntax": "hgvs.c"}]}, - hgvs_assay_level=hgvs, - ) - session.add(mapped_variant_2) + + sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_processed"] == 2 - assert result.data["variants_with_consequences"] == 2 + mock_resolve.assert_called_once() # force bypassed the current-release skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant_1) - session.refresh(mapped_variant_2) - assert mapped_variant_1.vep_functional_consequence == "missense_variant" - assert mapped_variant_2.vep_functional_consequence == "missense_variant" + # Unchanged consequence -> no supersede: one row, still live, access_date touched in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].access_date == date.today() - async def test_vep_batch_api_exception_raises( + async def test_new_release_changed_consequence_supersedes( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the VEP API propagates to the job management decorator.""" - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=RuntimeError("VEP API unreachable"), - ), - pytest.raises(RuntimeError, match="VEP API unreachable"), - ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; a changed result supersedes it — one live + row carrying the new consequence/release and one retired row preserving the old.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> not skipped, eligible for re-query. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() - async def test_variant_recoder_api_exception_raises( + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE + + # Old consequence retired, not deleted. + all_rows = session.scalars( + select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id) + ).all() + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + async def test_successful_linking_pipeline( + self, + session, + with_populated_domain_data, + mock_worker_ctx, + sample_populate_vep_run_pipeline, + sample_populate_vep_pipeline, + setup_sample_alleles_for_vep, + ): + """End-to-end successful linking within a pipeline updates both job and pipeline status.""" + _, allele = setup_sample_alleles_for_vep + + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) + + assert result.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_exceptions_handled_by_decorators( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the Variant Recoder API propagates to the job management decorator.""" + """Exceptions during resolution are handled by the job decorators (Slack alert, ERRORED).""" with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - side_effect=RuntimeError("Recoder API unreachable"), - ), - pytest.raises(RuntimeError, match="Recoder API unreachable"), + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_send_slack_job_error.assert_called_once() + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.ERRORED + assert isinstance(result.exception, Exception) + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED @pytest.mark.asyncio @pytest.mark.integration -class TestPopulateVepForScoreSetIntegration: - """Integration tests for populate_vep_for_score_set run through an ARQ worker context.""" +class TestPopulateVepForScoreSetArqContext: + """Tests for the populate_vep_for_score_set job using the ARQ context fixture.""" - async def test_populate_vep_with_arq_context( + async def test_populate_vep_with_arq_context_independent( self, arq_redis, arq_worker, @@ -449,35 +515,29 @@ async def test_populate_vep_with_arq_context( with_populated_domain_data, with_populate_vep_job, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes successfully within an ARQ worker context.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() - session.refresh(sample_populate_vep_run) - assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED - async def test_populate_vep_in_pipeline_context( + async def test_populate_vep_with_arq_context_pipeline( self, arq_redis, arq_worker, @@ -485,24 +545,73 @@ async def test_populate_vep_in_pipeline_context( with_populated_domain_data, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes and advances the pipeline when run in a pipeline context.""" - from mavedb.models.enums.job_pipeline import PipelineStatus - - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job completes and advances the pipeline through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "synonymous_variant"}, - ): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_populate_vep_with_arq_context_exception_handling_independent( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + with_populate_vep_job, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job are handled with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED + + async def test_populate_vep_with_arq_context_exception_handling_pipeline( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + sample_populate_vep_pipeline, + sample_populate_vep_run_pipeline, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job fail the pipeline with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 104c3af6d..084fe93d7 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -2639,14 +2639,16 @@ async def test_cleanup_arq_integration(self, arq_redis, arq_worker, standalone_w await arq_worker.async_run() # Don't call run_check() - the retried test_function doesn't exist and would fail - # Verify the cleanup job succeeded - cleanup_job = session.execute( - select(JobRun).where(JobRun.job_function == "cleanup_stalled_jobs") - ).scalar_one_or_none() - - assert cleanup_job is not None - assert cleanup_job.status == JobStatus.SUCCEEDED - assert cleanup_job.job_type == "cron_job" + # Verify the cleanup job succeeded. Asserted over every run rather than via + # `scalar_one_or_none`, because the `arq_worker` fixture registers `BACKGROUND_CRONJOBS` and + # `cleanup_stalled_jobs_cron` fires at minutes {5, 15, 25, 35, 45, 55}. A burst run that happens + # to straddle one of those minutes executes the cron alongside the job enqueued above, so the + # row count depends on the wall clock. Every run must still succeed, which is the property here. + cleanup_jobs = session.scalars(select(JobRun).where(JobRun.job_function == "cleanup_stalled_jobs")).all() + + assert cleanup_jobs + assert all(job.status == JobStatus.SUCCEEDED for job in cleanup_jobs) + assert all(job.job_type == "cron_job" for job in cleanup_jobs) # Verify the stalled job was cleaned up session.refresh(stalled_job) diff --git a/tests/worker/jobs/test_allele_translations.py b/tests/worker/jobs/test_allele_translations.py new file mode 100644 index 000000000..0e1c1b12c --- /dev/null +++ b/tests/worker/jobs/test_allele_translations.py @@ -0,0 +1,41 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from datetime import datetime, timezone + +from mavedb.lib.alleles import get_allele_translations + + +@pytest.mark.unit +class TestGetAlleleTranslations: + """The cross-layer equivalence query traverses the MappingRecordAllele link graph. + + Uses the RT-derived fixture, which links an authoritative allele and a derived (cross-layer) + allele to one MappingRecord — exactly the co-membership the query resolves. + """ + + def test_returns_full_equivalence_set_from_any_member(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + from_authoritative = {a.id for a in get_allele_translations(session, authoritative_allele.id)} + from_rt = {a.id for a in get_allele_translations(session, rt_allele.id)} + + # Reachable from either member, and includes the anchor itself — the full equivalence set. + assert from_authoritative == {authoritative_allele.id, rt_allele.id} + assert from_rt == {authoritative_allele.id, rt_allele.id} + + def test_as_of_before_links_existed_is_empty(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, _rt = setup_rt_derived_allele_with_caid + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + assert get_allele_translations(session, authoritative_allele.id, as_of=past) == [] + + def test_as_of_after_links_existed_returns_set(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + ids = {a.id for a in get_allele_translations(session, authoritative_allele.id, as_of=future)} + assert ids == {authoritative_allele.id, rt_allele.id} diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index dc2ac843e..d4134ab05 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -5,17 +5,22 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import date from unittest.mock import MagicMock, patch from sqlalchemy.exc import NoResultFound from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import TEST_CODING_LAYER, TEST_GENOMIC_LAYER, TEST_PROTEIN_LAYER @@ -24,6 +29,25 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def authoritative_allele_for(session, mapping_record): + """Return the authoritative (post-mapped) Allele linked to a mapping record, or None. + + Under the allele data model the post-mapped VRS representation lives on an + ``Allele`` joined to the ``MappingRecord`` through ``MappingRecordAllele``. + A successfully post-mapped variant has exactly one authoritative link; a + variant that failed to post-map has a mapping record but no such link. + """ + return ( + session.query(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .filter( + MappingRecordAllele.mapping_record_id == mapping_record.id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .one_or_none() + ) + + @pytest.mark.unit @pytest.mark.asyncio class TestMapVariantsForScoreSetUnit: @@ -66,8 +90,8 @@ async def test_map_variants_for_score_set_no_mapping_results( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -90,7 +114,10 @@ async def test_map_variants_for_score_set_no_mapped_scores( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [], "error_message": "No variants were mapped for this score set"} + { + "mapped_scores": [], + "error_message": "No variants were mapped for this score set", + } ), ), ): @@ -110,8 +137,8 @@ async def test_map_variants_for_score_set_no_mapped_scores( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -134,7 +161,10 @@ async def test_map_variants_for_score_set_no_reference_data( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [MagicMock()], "error_message": "Reference metadata missing from mapping results"} + { + "mapped_scores": [MagicMock()], + "error_message": "Reference metadata missing from mapping results", + } ), ), ): @@ -154,8 +184,8 @@ async def test_map_variants_for_score_set_no_reference_data( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -209,8 +239,8 @@ async def test_map_variants_for_score_set_nonexistent_target_gene( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -256,8 +286,8 @@ async def test_map_variants_for_score_set_returns_variants_not_in_score_set( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -318,20 +348,20 @@ async def dummy_mapping_job(): for target in sample_score_set.target_genes: assert target.mapped_hgnc_name is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" @pytest.mark.parametrize( "with_layers", @@ -435,20 +465,64 @@ async def dummy_mapping_job(): else: assert target.post_mapped_metadata.get("protein") is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" + + async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """An identity cdna TargetGeneMapping (a cdna layer with no per-variant scores + joining it) persists with its reference_accession (NM_) and null QC/counts -- the + artifact reverse translation consumes to resolve the projection transcript.""" + variant = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + + # Genomic assay layer + an identity cdna layer (no cdna per-variant scores). + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, score_set=sample_score_set, with_layers={"g", "c"} + ) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + assert result.status == JobStatus.SUCCEEDED + + cdna_tgms = ( + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == SequenceLevel.cdna).all() + ) + assert len(cdna_tgms) == len(sample_score_set.target_genes) + tgm = cdna_tgms[0] + assert tgm.reference_accession == "NM_999999.1" + # Identity row: no mapped_variant joins it, so QC/counts are null. + assert tgm.total_variants is None + assert tgm.variants_mapped_cleanly is None + assert tgm.percent_identity is None async def test_map_variants_for_score_set_success_no_successful_mapping( self, @@ -501,24 +575,23 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.failed assert sample_score_set.mapping_errors["error_message"] == "All variants failed to map." - # Verify that one mapped variant was created. Although no successful mapping, an entry is still created. - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that one mapping record was created. Although no successful mapping, an entry is still created. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 - # Verify that the mapped variant has no post-mapped data - mapped_variant = mapped_variants[0] - assert mapped_variant.post_mapped == {} + # Verify that the mapping record has no authoritative (post-mapped) allele + assert authoritative_allele_for(session, mapping_records[0]) is None # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "failed" + assert annotation_statuses[0].disposition == "failed" async def test_map_variants_for_score_set_incomplete_mapping( self, @@ -584,38 +657,124 @@ async def dummy_mapping_job(): # Although only one variant was successfully mapped, verify that an entity was created # for each variant in the score set - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 - - # Verify that only one variant has post-mapped data - mapped_variant_with_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped != {}).one_or_none() - ) - assert mapped_variant_with_post_data is not None + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 - mapped_variant_without_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped == {}).one_or_none() - ) - assert mapped_variant_without_post_data is not None + # Verify that exactly one mapping record has post-mapped (authoritative) allele data + records_with_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is not None] + records_without_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is None] + assert len(records_with_post_data) == 1 + assert len(records_without_post_data) == 1 # Verify that annotation statuses were created and correct annotation_status_success = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "success") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "present") .all() ) assert len(annotation_status_success) == 1 assert annotation_status_success[0].annotation_type == "vrs_mapping" annotation_status_failed = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "failed") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "failed") .all() ) assert len(annotation_status_failed) == 1 assert annotation_status_failed[0].annotation_type == "vrs_mapping" + async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A score set whose only unmapped variants are benign absences (intronic / no + protein consequence) is ``complete``, not ``incomplete``/``failed``: benign + outcomes carry no allele, so they are recorded as ``absent`` (an informative + biological negative), never ``failed``.""" + + async def dummy_mapping_job(): + mapping_output = await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + # Re-stamp the emitted records as benign absences: no allele, no failure. The + # helper only produces MAPPED/FAILED, so we override here to exercise the path. + benign_outcomes = ["intronic", "no_protein_consequence"] + for idx, mapped_score in enumerate(mapping_output["mapped_scores"]): + mapped_score["pre_mapped"] = {} + mapped_score["post_mapped"] = {} + mapped_score["outcome"] = benign_outcomes[idx % len(benign_outcomes)] + return mapping_output + + variant1 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + urn="variant:1", + ) + variant2 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + urn="variant:2", + ) + session.add_all([variant1, variant2]) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + # The run succeeded and the score set is complete -- nothing genuinely failed. + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["mapped_count"] == 0 + assert result.data["failed_count"] == 0 + assert result.data["skipped_count"] == 2 + assert sample_score_set.mapping_state == MappingState.complete + assert sample_score_set.mapping_errors is None + + # A record exists per variant, but with no authoritative allele. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 + assert all(authoritative_allele_for(session, r) is None for r in mapping_records) + + # Benign outcomes carry no allele → recorded as `absent` (not `failed`), with the finer + # outcome preserved as the event `reason`. + events = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id) + .all() + ) + assert len(events) == 2 + assert all(e.disposition == "absent" for e in events) + recorded_outcomes = {e.reason for e in events} + assert recorded_outcomes == {"intronic", "no_protein_consequence"} + async def test_map_variants_for_score_set_complete_mapping( self, session, @@ -678,29 +837,29 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. for urn in ["variant:1", "variant:2"]: - mapped_variant = session.query(MappedVariant).filter(MappedVariant.variant.has(urn=urn)).one_or_none() - assert mapped_variant is not None - assert mapped_variant.post_mapped != {} - assert mapped_variant.hgvs_assay_level is not None + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant.has(urn=urn)).one_or_none() + assert mapping_record is not None + assert authoritative_allele_for(session, mapping_record) is not None + assert mapping_record.hgvs_assay_level is not None # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 2 for status in annotation_statuses: assert status.annotation_type == "vrs_mapping" - assert status.status == "success" + assert status.disposition == "present" async def test_map_variants_for_score_set_updates_existing_mapped_variants( self, @@ -733,18 +892,28 @@ async def dummy_mapping_job(): ) session.add(variant) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=SequenceLevel.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - variant_annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + # Link the prior record to an authoritative allele. Re-mapping must retire this link too, + # not just the record — a live link dangling off a retired record breaks the temporal model. + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=SequenceLevel.genomic.value) + session.add(prior_allele) + session.commit() + prior_link = MappingRecordAllele( + mapping_record_id=mapping_record.id, allele_id=prior_allele.id, is_authoritative=True + ) + session.add(prior_link) + session.commit() + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(variant_annotation_status) + session.add(prior_event) session.commit() with ( @@ -766,44 +935,45 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify the existing mapped variant was marked as non-current - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.id == mapped_variant.id, MappedVariant.current.is_(False)) + # Verify the existing mapping record was retired (valid_to closed) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.id == mapping_record.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None + + # Verify the prior record's authoritative allele link was retired alongside the record, + # so no live link dangles off the retired record. + session.refresh(prior_link) + assert prior_link.valid_to is not None - # Verify a new mapped variant entry was created - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + # Verify a new, live mapping record entry was created + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Gap-free handoff: the prior record closed at exactly the new record's valid_from (one + # supersession timestamp), and the cascade closed the prior link at that same instant — so + # no point-in-time query lands in a hole between the old and new record or their links. + assert non_current_mapping_record.valid_to == new_mapping_record.valid_from + assert prior_link.valid_to == non_current_mapping_record.valid_to - # Verify the non-current annotation status still exists - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter( - VariantAnnotationStatus.variant_id == non_current_mapped_variant.variant_id, - VariantAnnotationStatus.current.is_(False), - ) - .one_or_none() - ) - assert old_annotation_status is not None + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that a new annotation status was created - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + # Append-only: the prior event is retained and the re-map appends a new one, so the variant + # now has two vrs_mapping events (there is no current flag to flip). + events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(events) == 2 @pytest.mark.integration @@ -862,9 +1032,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -875,19 +1045,19 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -953,9 +1123,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -966,19 +1136,19 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1053,12 +1223,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1130,12 +1300,12 @@ async def dummy_mapping_job(): # Error message originates from our mock mapping construction function assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1206,12 +1376,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "Reference metadata missing from mapping results" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1246,20 +1416,20 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( sample_independent_variant_creation_run, ) - # Associate mapped variants with all variants just created in the score set + # Associate mapping records with all variants just created in the score set variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() for variant in variants: - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=SequenceLevel.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(annotation_status) - session.add(mapped_variant) + session.add(prior_event) + session.add(mapping_record) session.commit() async def dummy_mapping_job(): @@ -1295,45 +1465,38 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were marked as non-current and new entries created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == len(variants) * 2 # Each variant has two mapped entries now + # Verify that mapping records were marked as non-current and new entries created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == len(variants) * 2 # Each variant has two mapping records now for variant in variants: - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(False)) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that annotation statuses where marked as non-current and new entries created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == len(variants) * 2 # Each variant has two annotation statuses now + # Append-only: each variant keeps its prior event and gains a new one from this run. + annotation_events = session.query(AnnotationEvent).all() + assert len(annotation_events) == len(variants) * 2 for variant in variants: - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(False)) - .one_or_none() - ) - assert old_annotation_status is not None - - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + variant_events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(variant_events) == 2 # Verify that the job status was updated. processing_run = ( @@ -1389,12 +1552,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1448,12 +1611,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1518,27 +1681,27 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1606,27 +1769,27 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1690,12 +1853,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1744,12 +1907,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py new file mode 100644 index 000000000..030d17020 --- /dev/null +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -0,0 +1,1501 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +import contextlib +from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import timedelta +from unittest.mock import MagicMock, patch + +from variant_annotation.lib.translation.types import ( + ProjectionPair, + TranslationError, + TranslationResult, + WtCodonMode, +) + +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.target_category import TargetCategory +from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.worker.jobs.variant_processing.reverse_translation import ( + _build_translation_config, + reverse_translate_variants_for_score_set, +) +from mavedb.worker.lib.managers.job_manager import JobManager +from tests.helpers.constants import TEST_GA4GH_IDENTIFIER +from tests.helpers.util.setup.worker import construct_mock_mapping_output + +pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") + +# Module under test, used as the patch root for the external annotation-package calls. +RT_MODULE = "mavedb.worker.jobs.variant_processing.reverse_translation" + + +class FakeVrsVariation: + """Stand-in for a ga4gh.vrs variation returned by translate_hgvs_to_variation. + + The reverse translation job only reads ``.id`` (for the VRS digest / dedup key) + and ``.model_dump()`` (persisted into the ``post_mapped`` JSONB column), so the + fake exposes exactly those. ``vrs_type`` lets a test assert whether a candidate + became a single Allele or a CisPhasedBlock. + """ + + def __init__(self, vrs_id: str, vrs_type: str = "Allele"): + self.id = vrs_id + self.vrs_type = vrs_type + + def model_dump(self, **kwargs) -> dict: + # Accept (and ignore) model_dump kwargs like exclude_none, mirroring the real VRS model. + return {"type": self.vrs_type, "id": self.id} + + +def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``construct_equivalent_variants``. + + The job correlates each TranslationResult/TranslationError back to its originating + mapping record via object identity on ``result.input``, so the fake must echo the + exact VariantInput instances it was handed. Results are keyed by the input's + assay-level HGVS string. + + Each result spec is one of: + + * a list of ``(hgvs_c, hgvs_g)`` pairs — one ProjectionPair each. + ``hgvs_g`` may be ``None`` for a projection-failed, coding-only projection pair. + * a ``(pairs, hgvs_p)`` tuple pairing that list with a shared protein apex. + + Each pair becomes one ProjectionPair, mirroring the position-aligned + coding↔genomic rows the real reverse-translate CLI emits (with ``--one-row-per-input``). + """ + errors_by_hgvs = errors_by_hgvs or {} + + def _construct(inputs, *, transcripts, coordinates, config): + results, errors = [], [] + for inp in inputs: + if inp.hgvs in errors_by_hgvs: + errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) + elif inp.hgvs in results_by_hgvs: + spec = results_by_hgvs[inp.hgvs] + # A bare list is the candidate pairs; a tuple pairs them with a protein apex. + pairs, hgvs_p = (spec[0], spec[1]) if isinstance(spec, tuple) else (spec, None) + results.append( + TranslationResult( + input=inp, + projection_pairs=[ProjectionPair(hgvs_c=hgvs_c, hgvs_g=hgvs_g) for hgvs_c, hgvs_g in pairs], + hgvs_p=hgvs_p, + ) + ) + else: + errors.append(TranslationError(input=inp, error=f"no stub result for {inp.hgvs!r}")) + return results, errors + + return _construct + + +def fake_translate(id_by_hgvs: dict, type_by_hgvs: dict | None = None, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``translate_hgvs_to_variation`` mapping candidate HGVS -> VRS id. + + Unknown HGVS get a deterministic id derived from the string so distinct candidates + stay distinct; supplying explicit ids lets a test force two candidates to collapse. + ``type_by_hgvs`` lets a test mark a bracketed candidate as a ``CisPhasedBlock`` (the + real translate_hgvs_to_variation returns one for cis-phased multivariants). + ``errors_by_hgvs`` makes a candidate raise, standing in for an HGVS form ga4gh cannot + translate to VRS. + """ + type_by_hgvs = type_by_hgvs or {} + errors_by_hgvs = errors_by_hgvs or {} + + def _translate(hgvs: str, translator=None) -> FakeVrsVariation: + if hgvs in errors_by_hgvs: + raise ValueError(errors_by_hgvs[hgvs]) + return FakeVrsVariation( + id_by_hgvs.get(hgvs, f"ga4gh:VA.{hgvs}"), + type_by_hgvs.get(hgvs, "Allele"), + ) + + return _translate + + +async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): + """Run the (mocked) mapping job so the score set's variants gain authoritative + MappingRecords/Alleles — the real inputs the reverse translation job reads.""" + + async def dummy_mapping_job(): + return await construct_mock_mapping_output(session=session, score_set=score_set, with_layers=with_layers) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], mapping_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.commit() + + +async def _reverse_translate(session, mock_worker_ctx, rt_run): + # The job opens a UTA-backed TranscriptSource to back codon_at (WtCodonMode.ALL). + # construct_equivalent_variants is mocked in these tests, so the source is never + # queried -- stub the factory with a no-op context yielding a dummy client. + with patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) + + +def _cross_level_events(session, score_set_id, disposition=None, reason=None): + query = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter( + Variant.score_set_id == score_set_id, + AnnotationEvent.annotation_type == "cross_level_translation", + ) + ) + if disposition is not None: + query = query.filter(AnnotationEvent.disposition == disposition) + if reason is not None: + query = query.filter(AnnotationEvent.reason == reason) + return query.all() + + +def _non_authoritative_links(session): + return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() + + +def _live_links(session, record_id): + """Every live (valid_to IS NULL) link for a mapping record, authoritative or derived.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.valid_to.is_(None), + ) + .all() + ) + + +def _authoritative_link(session, record_id): + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .one() + ) + + +def _record_for(session, variant_id): + return session.query(MappingRecord).filter(MappingRecord.variant_id == variant_id).one() + + +@pytest.mark.unit +class TestBuildTranslationConfig: + """Unit tests for _build_translation_config — the optional translation_config job param.""" + + def test_none_uses_job_defaults(self): + config = _build_translation_config(None) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_empty_dict_uses_job_defaults(self): + config = _build_translation_config({}) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_overrides_win_and_string_wt_codon_mode_is_coerced(self): + config = _build_translation_config({"wt_codon_mode": "unambiguous", "max_indel_size": 7}) + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + assert config.include_indels is True # untouched default preserved + + def test_can_disable_indels_with_none_mode(self): + config = _build_translation_config({"wt_codon_mode": "none", "include_indels": False}) + assert config.wt_codon_mode == WtCodonMode.NONE + assert config.include_indels is False + + def test_rejects_invalid_wt_codon_mode_with_actionable_message(self): + with pytest.raises(ValueError, match=r"wt_codon_mode 'bogus'.*Valid values"): + _build_translation_config({"wt_codon_mode": "bogus"}) + + def test_rejects_wt_codon_mode_without_indels(self): + # TranslationConfig enforces: a wt_codon_mode other than "none" requires include_indels. + with pytest.raises(ValueError, match="translation_config"): + _build_translation_config({"wt_codon_mode": "all", "include_indels": False}) + + def test_rejects_unknown_field_and_lists_valid_options(self): + with pytest.raises( + ValueError, match=r"Unknown translation_config option\(s\): not_a_real_field.*Valid options" + ): + _build_translation_config({"not_a_real_field": 1}) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestReverseTranslateVariantsForScoreSetUnit: + """Unit tests for the reverse_translate_variants_for_score_set job.""" + + async def test_no_mapping_records_is_a_noop( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no current, post-mapped mapping records there is nothing to translate.""" + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0} + + # No candidate alleles, links, or annotations were produced. + assert session.query(Allele).count() == 0 + assert _non_authoritative_links(session) == [] + assert _cross_level_events(session, sample_score_set.id) == [] + + async def test_creates_genomic_and_coding_candidate_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A mapped variant expands into one non-authoritative allele per equivalence-class + candidate, each linked to the originating mapping record. The two members of a single + projection pair (coding + genomic projection) share one projection_group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + + # Two non-authoritative links were created, both for our one mapping record. + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 2 + + genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() + assert genomic_allele.level == SequenceLevel.genomic.value + assert genomic_allele.hgvs_g == g_candidate + assert genomic_allele.hgvs_c is None + assert genomic_allele.transcript == "NC_000001.11" + assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} + + coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() + assert coding_allele.level == SequenceLevel.cdna.value + assert coding_allele.hgvs_c == c_candidate + assert coding_allele.hgvs_g is None + assert coding_allele.transcript == "NM_000001.1" + + # The candidate alleles are linked to the same mapping record as the authoritative allele. + mapping_record = _record_for(session, variant.id) + assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + + # Both members of the single projection pair carry the same (first, index 0) group id — the + # coding↔genomic pairing survives the flatten that Phase B exists to stop. + assert {link.projection_group for link in non_auth_links} == {0} + + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 1 + assert events[0].disposition == "present" + # The event metadata carries the paired projection pair shape, not the removed flat lists. + assert events[0].event_metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None} + ] + + async def test_transcript_is_queryable_via_derived_expression( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Allele.transcript is a derived hybrid_property — filtering on it in SQL resolves to + the accession of the populated HGVS column, with no stored transcript column.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The SQL expression derives the accession from the populated HGVS column. + genomic = session.query(Allele).filter(Allele.transcript == "NC_000001.11").all() + assert [a.vrs_digest for a in genomic] == ["ga4gh:VA.genomic"] + coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() + assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + + async def test_protein_consequence_is_persisted_as_a_protein_allele( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The deterministic protein consequence (``result.hgvs_p``) is emitted as the + protein-level member of the equivalence set -- a non-authoritative ``level=protein`` + allele linked to the record, not dropped. As the apex shared across all projection pairs it + belongs to no projection pair, so its link's projection_group is NULL.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + # c_to_p emits a predicted consequence in parens; the job strips them before use. + p_consequence = "NP_000001.1:p.(Met1Val)" + p_stripped = "NP_000001.1:p.Met1Val" + construct = fake_construct({assay_hgvs: ([(c_candidate, g_candidate)], p_consequence)}) + translate = fake_translate( + { + g_candidate: "ga4gh:VA.genomic", + c_candidate: "ga4gh:VA.coding", + p_stripped: "ga4gh:VA.protein", # the stripped form is what reaches the translator + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() + assert protein_allele.level == SequenceLevel.protein.value + assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens + + # Linked to the record as a non-authoritative member of the equivalence set, but grouped + # with no pair (the apex is shared across every projection pair). + protein_link = next(link for link in _non_authoritative_links(session) if link.allele_id == protein_allele.id) + assert protein_link.projection_group is None + # The coding/genomic pair is still grouped together (group 0). + pair_links = [link for link in _non_authoritative_links(session) if link.allele_id != protein_allele.id] + assert {link.projection_group for link in pair_links} == {0} + + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A bracketed genomic projection (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest, and + grouped with its coding partner in the projection pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" + construct = fake_construct({assay_hgvs: [(c_candidate, block_candidate)]}) + translate = fake_translate( + {c_candidate: "ga4gh:VA.coding", block_candidate: "ga4gh:CPB.block"}, + type_by_hgvs={block_candidate: "CisPhasedBlock"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + + block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() + assert block_allele.level == SequenceLevel.genomic.value + # The full bracketed expression is stored verbatim; its accession anchors the row. + assert block_allele.hgvs_g == block_candidate + assert block_allele.transcript == "NC_000001.11" + assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} + + # The block (genomic projection) and its coding partner are one projection pair -> one group. + assert {link.projection_group for link in _non_authoritative_links(session)} == {0} + + async def test_duplicate_candidate_digests_are_deduped_per_record( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When the coding and genomic members of a projection pair resolve to the same VRS digest, + only one allele/link is written for that mapping record (per the library's dedup contract). + The surviving link still carries the pair's group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + # Both members of the projection pair collapse to the same VRS digest. + translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 1 + assert non_auth_links[0].projection_group == 0 + assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 + + async def test_authoritative_allele_is_folded_into_its_projection_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Nucleotide assay: the measured allele appears among the RT candidates (the library makes + no exclusions) and already has a live authoritative link (group NULL). Rather than insert a + forbidden derived duplicate, the job folds the pair's group onto the existing + authoritative link. That group then resolves to {canonical coding, canonical genomic}: the + authoritative allele plus its sibling projection derived in the same projection pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Exactly one allele exists after mapping: the authoritative (measured) allele. + assert session.query(Allele).count() == 1 + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member IS the measured allele (folds in); the genomic member is its projection. + translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER, g_sibling: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # Only the genomic sibling is a newly created derived allele; the coding member folds into + # the existing authoritative link rather than creating a second link/allele. + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert len(derived_links) == 1 + + # The authoritative link was folded into the pair's group, and the sibling derived + # link shares it -- the pairing is resolvable from the authoritative allele's group. + assert auth_link.projection_group == 0 + assert derived_links[0].projection_group == 0 + + # group 0 resolves to {canonical c (authoritative), canonical g (projection)}: two live + # links, one authoritative and one derived. + group_links = [link for link in _live_links(session, record.id) if link.projection_group == 0] + assert len(group_links) == 2 + assert {link.is_authoritative for link in group_links} == {True, False} + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" + + async def test_authoritative_fold_in_when_measured_at_genomic_level( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The fold-in is symmetric in the measured level: when the genomic member of a projection pair + equals the record's authoritative allele, its group folds onto the authoritative link and + the coding sibling becomes the derived member of the same group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity mapping so RT resolves a coding transcript. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_sibling = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_sibling, g_candidate)]}) + # The genomic member IS the measured allele (folds in); the coding member is its projection. + translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: TEST_GA4GH_IDENTIFIER}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert auth_link.projection_group == 0 + assert [link.projection_group for link in derived_links] == [0] + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" + + async def test_projection_failed_candidate_is_a_one_member_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A projection pair whose genomic projection failed (hgvs_g is None) yields a well-formed + one-member (coding-only) group -- no crash, no desync, and the group holds just the coding + link. This is the case the old two-list shape silently dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_paired = "NM_000001.1:c.5A>G" + g_paired = "NC_000001.11:g.1000A>G" + c_orphan = "NM_000001.1:c.9A>T" # projection failed: no genomic partner + construct = fake_construct({assay_hgvs: [(c_paired, g_paired), (c_orphan, None)]}) + translate = fake_translate( + { + c_paired: "ga4gh:VA.coding0", + g_paired: "ga4gh:VA.genomic0", + c_orphan: "ga4gh:VA.coding1", + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3} + + non_auth_links = _non_authoritative_links(session) + groups: dict[int, set[str]] = {} + for link in non_auth_links: + groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + + # Group 0 is the full projection pair; group 1 is the projection-failed coding-only member. + assert groups == { + 0: {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, + 1: {"ga4gh:VA.coding1"}, + } + + async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Protein assay: the authoritative allele is the protein itself, which appears in no + coding/genomic projection pair, so nothing folds in and its link's group stays NULL. The + ambiguous fan-out becomes N distinct groups, each a coding/genomic pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: the authoritative allele is the protein allele. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NP_000000.1:p.Met1Val" + # A degenerate protein consequence -> three coding/genomic projection pairs (the fan-out). + pairs = [ + ("NM_000001.1:c.3A>G", "NC_000001.11:g.100A>G"), + ("NM_000001.1:c.3A>C", "NC_000001.11:g.100A>C"), + ("NM_000001.1:c.3A>T", "NC_000001.11:g.100A>T"), + ] + # Protein assay: no protein apex is re-emitted (result.hgvs_p is None) -- the protein + # allele is already authoritative. + construct = fake_construct({assay_hgvs: pairs}) + translate = fake_translate({hgvs: f"ga4gh:VA.{hgvs}" for pair in pairs for hgvs in pair}) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000001.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6} + + record = _record_for(session, variant.id) + # The protein authoritative allele is not part of any pair: its group stays NULL. + auth_link = _authoritative_link(session, record.id) + assert auth_link.projection_group is None + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + + # Three distinct groups, each a coding/genomic pair (two members). + groups: dict[int, int] = {} + for link in _non_authoritative_links(session): + groups[link.projection_group] = groups.get(link.projection_group, 0) + 1 + assert groups == {0: 2, 1: 2, 2: 2} + + async def test_independent_rerun_retires_prior_links_and_regroups( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An independent second run is idempotent: it retires the prior derived links (closing + valid_to) and writes fresh live ones (re-grouped from scratch), so the set of *current* + non-authoritative links is stable while the superseded links are retained as history -- + the partial unique index on live links is never violated and no alleles are duplicated. + projection_group is regenerated per run; it is a within-record grouping key, not a stable + identity across re-maps.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + # First run: two live derived links, both in group 0. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert first.status == JobStatus.SUCCEEDED + first_links = _non_authoritative_links(session) + assert len(first_links) == 2 + assert all(link.valid_to is None for link in first_links) + assert {link.projection_group for link in first_links} == {0} + + # Second, independent run with identical inputs. + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + + assert second.status == JobStatus.SUCCEEDED + + links = _non_authoritative_links(session) + live = [link for link in links if link.valid_to is None] + retired = [link for link in links if link.valid_to is not None] + + # The live set is stable (still exactly the two candidates, re-grouped into group 0) and the + # prior run's links are retained as closed history rather than deleted. + assert len(live) == 2 + assert len(retired) == 2 + assert {link.projection_group for link in live} == {0} + + # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under + # one supersession timestamp -- so a point-in-time query never lands in a hole between runs, + # even though the translation loop ran (and could have committed) between retire and insert. + assert {link.valid_to for link in retired} == {link.valid_from for link in live} + assert len({link.valid_from for link in live}) == 1 + + # The two candidate alleles are reused across runs, not duplicated. + assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 + + # The cross-level translation log is append-only: the rerun adds a second event, and + # "current" is the latest by id (there is no current flag to desync). + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 2 + assert max(events, key=lambda e: e.id).disposition == "present" + + async def test_all_failures_fail_the_job( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """If every variant's translation errors, the job fails and records FAILED annotations.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct({}, errors_by_hgvs={assay_hgvs: "forward translation failed"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + + assert _non_authoritative_links(session) == [] + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "forward translation failed" + # Library/input-level failures still carry the assay-level HGVS as metadata. + assert failed_events[0].event_metadata["hgvs_input"] == assay_hgvs + + async def test_partial_success_succeeds_with_mixed_annotations( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One variant translating and another erroring yields a SUCCEEDED job with both a + success and a failure annotation.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + variant_err = Variant( + score_set_id=sample_score_set.id, + urn="variant:2", + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + ) + session.add_all([variant_ok, variant_err]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, + ) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + + assert len(_cross_level_events(session, sample_score_set.id, disposition="present")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 + assert len(_non_authoritative_links(session)) == 1 + + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One member of a projection pair translating and the other failing VRS translation leaves the + variant a SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + good_coding = "NM_000001.1:c.5A>G" + bad_genomic = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: [(good_coding, bad_genomic)]}) + translate = fake_translate( + {good_coding: "ga4gh:VA.coding"}, + errors_by_hgvs={bad_genomic: "untranslatable form"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert len(_non_authoritative_links(session)) == 1 + + events = _cross_level_events(session, sample_score_set.id, disposition="present") + assert len(events) == 1 + failed_candidates = events[0].event_metadata["failed_candidates"] + assert failed_candidates == [{"hgvs": bad_genomic, "level": "genomic", "error": "untranslatable form"}] + + async def test_all_candidates_failing_translation_marks_variant_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When every candidate fails VRS translation, the variant is FAILED, no allele is + created, and the per-candidate errors are retained in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + bad_candidate = "NM_000001.1:c.9A>T" # a coding-only projection pair whose sole member fails + construct = fake_construct({assay_hgvs: [(bad_candidate, None)]}) + translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + assert _non_authoritative_links(session) == [] + + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "All candidate HGVS failed VRS translation." + metadata = failed_events[0].event_metadata + assert metadata["hgvs_input"] == assay_hgvs + assert metadata["failed_candidates"] == [ + {"hgvs": bad_candidate, "level": "cdna", "error": "untranslatable form"} + ] + + async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target aligned only at the genomic level has no resolvable coding + transcript (transcript_unresolved). The job tallies it in its `skipped` counter, but the + recoverable gap is recorded as a `failed` event — distinct from a hard translation + failure (reason `translation_failed`) and from a true biological absence + (no_coding_transcript → `absent`).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript to anchor on. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + + # No translation attempted: no candidate alleles or links, and the variant is + # recorded as SKIPPED rather than FAILED. + assert _non_authoritative_links(session) == [] + # The job counts this as a skip, but a protein-coding target with no resolvable coding + # transcript is a *recoverable* gap (transcript_unresolved) → recorded as a `failed` event, + # distinct from a hard translation failure (which carries reason `translation_failed`). + unresolved = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(unresolved) == 1 + assert unresolved[0].disposition == "failed" + + async def test_genomic_accession_coding_target_is_reverse_translated( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A genomic-accession (NC_:g.) coding variant -- previously skipped for want of a + coding transcript -- is reverse-translated once the mapper emits a cdna + TargetGeneMapping, whose reference_accession anchors the projection.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity TargetGeneMapping (carrying the NM_); the + # genomic variant projects onto that transcript for reverse translation. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + with ( + patch( + f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + ), + patch( + f"{RT_MODULE}.translate_hgvs_to_variation", + fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}), + ), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + events = _cross_level_events(session, sample_score_set.id) + assert events and all(e.disposition == "present" for e in events) + + def _run_mapped_date(self, session, target_gene_id): + """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" + return ( + session.query(TargetGeneMapping) + .filter( + TargetGeneMapping.target_gene_id == target_gene_id, + TargetGeneMapping.alignment_level == SequenceLevel.genomic, + ) + .first() + .mapped_date + ) + + async def test_uses_latest_cdna_row_within_the_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a run has more than one cdna TargetGeneMapping for a target (same run date), + RT reverse-translates against the newest (highest id), not an arbitrary one.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with the run date. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A newer cdna row (higher id) for the same target and same run date. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=SequenceLevel.cdna, + reference_accession="NM_111111.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date, + ) + ) + session.commit() + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.g"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # The newest cdna row within the run wins, not the first-mapped one. + assert captured["transcripts"] == ["NM_111111.1"] + + async def test_ignores_stale_cdna_row_from_a_different_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A cdna row left behind by a *different* run (different run date) is not used: + the current run emitted no cdna row, so the variant is skipped (transcript + unresolved) rather than reverse-translated against the stale transcript -- the + mapped_date anchor narrows the accumulation edge case (mavedb-api#763).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Current run maps genomic only -- no cdna row for this run's date. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A leftover cdna row from a *prior* run (an earlier date) must not be picked up. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=SequenceLevel.cdna, + reference_accession="NM_888888.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date - timedelta(days=1), + ) + ) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(skipped) == 1 + assert skipped[0].disposition == "failed" + + async def test_coding_target_skip_is_classified_recoverable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target with no resolvable coding transcript is a *recoverable* + skip -- classified ``transcript_unresolved`` so it is findable, distinct from a + regulatory target's correct skip.""" + assert sample_score_set.target_genes[0].category == TargetCategory.protein_coding + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript resolves. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(skipped) == 1 + assert skipped[0].disposition == "failed" + + async def test_regulatory_target_skip_is_classified_correct( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A non-coding/regulatory target has no protein consequence: its skip is *correct*, + classified ``no_coding_transcript`` rather than the recoverable category.""" + target = sample_score_set.target_genes[0] + target.category = TargetCategory.regulatory + session.add(target) + session.commit() + + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="no_coding_transcript") + assert len(skipped) == 1 + assert skipped[0].disposition == "absent" + + async def test_translation_config_param_overrides_job_defaults( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A translation_config job param overrides the job's TranslationConfig defaults, and the + resulting config is passed through to construct_equivalent_variants.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + run = sample_independent_reverse_translation_run + run.job_params = {**run.job_params, "translation_config": {"wt_codon_mode": "unambiguous", "max_indel_size": 7}} + session.add(run) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + # Defaults the param did not override are preserved. + assert config.include_indels is True + + async def test_translation_config_defaults_when_param_absent( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no translation_config param the job falls back to its sensible defaults + (full codon equivalence class, indels included).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + async def test_protein_level_transcript_resolved_via_uta( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-only mapping has no cdna alignment transcript, so its coding transcript + is resolved NP_→NM_ via UTA and supplied as the VariantInput hint.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: no cdna TargetGeneMapping, so the transcript must come from UTA. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + + assay_hgvs = "NP_000000.1:p.Met1Val" + c_candidate = "NM_000111.1:c.3A>G" + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + # Capture the transcript hint each VariantInput was built with, then delegate to the + # normal stub for the translation result. + captured_hints: dict[str, str | None] = {} + delegate = fake_construct({assay_hgvs: [(c_candidate, None)]}) + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000111.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + # The NP_ accession was resolved to its NM_ transcript and passed as the hint. + assert captured_hints == {assay_hgvs: "NM_000111.1"}