Variant Search

.github/workflows/run-tests-on-push.yml

@@ -7,8 +7,8 @@ env:
 
 jobs:
   run-tests-3_9:
-    runs-on: ubuntu-20.04
-    name: Pytest on Python 3.9 / Ubuntu 20.04
+    runs-on: ubuntu-latest
+    name: Pytest on Python 3.9
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5

alembic/versions/34026092c7f8_clinvar_variant_table.py

@@ -0,0 +1,58 @@
+"""clinvar variant table
+
+Revision ID: 34026092c7f8
+Revises: e8a3b5d8f885
+Create Date: 2025-01-28 21:48:42.532346
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "34026092c7f8"
+down_revision = "e8a3b5d8f885"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "clinvar_variants",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("allele_id", sa.Integer(), nullable=False),
+        sa.Column("gene_symbol", sa.String(), nullable=False),
+        sa.Column("clinical_significance", sa.String(), nullable=False),
+        sa.Column("clinical_review_status", sa.String(), nullable=False),
+        sa.Column("clinvar_db_version", sa.String(), nullable=False),
+        sa.Column("creation_date", sa.Date(), nullable=False),
+        sa.Column("modification_date", sa.Date(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_clinvar_variants_allele_id"), "clinvar_variants", ["allele_id"], unique=False)
+    op.add_column("mapped_variants", sa.Column("clinvar_variant_id", sa.Integer(), nullable=True))
+    op.create_index(
+        op.f("ix_mapped_variants_clinvar_variant_id"), "mapped_variants", ["clinvar_variant_id"], unique=False
+    )
+    op.create_foreign_key(
+        "mapped_variant_clinvar_variant_id_foreign_key_constraint",
+        "mapped_variants",
+        "clinvar_variants",
+        ["clinvar_variant_id"],
+        ["id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "mapped_variant_clinvar_variant_id_foreign_key_constraint", "mapped_variants", type_="foreignkey"
+    )
+    op.drop_index(op.f("ix_mapped_variants_clinvar_variant_id"), table_name="mapped_variants")
+    op.drop_column("mapped_variants", "clinvar_variant_id")
+    op.drop_index(op.f("ix_clinvar_variants_allele_id"), table_name="clinvar_variants")
+    op.drop_table("clinvar_variants")
+    # ### end Alembic commands ###

alembic/versions/695b73abe581_genericize_clinvar_variants_table.py

@@ -0,0 +1,101 @@
+"""genericize clinvar variants table
+
+Revision ID: 695b73abe581
+Revises: 34026092c7f8
+Create Date: 2025-02-18 11:54:15.243078
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "695b73abe581"
+down_revision = "34026092c7f8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.rename_table("clinvar_variants", "clinical_controls")
+    op.execute("ALTER SEQUENCE clinvar_variants_id_seq RENAME TO clinical_controls_id_seq")
+    op.execute("ALTER INDEX clinvar_variants_pkey RENAME TO clinical_controls_pkey")
+
+    op.alter_column("clinical_controls", "clinvar_db_version", nullable=False, new_column_name="db_version")
+    op.alter_column("clinical_controls", "allele_id", nullable=False, new_column_name="db_identifier")
+    op.add_column("clinical_controls", sa.Column("db_name", sa.String(), nullable=True))
+
+    op.create_index("ix_clinical_controls_gene_symbol", "clinical_controls", ["gene_symbol"])
+    op.create_index("ix_clinical_controls_db_name", "clinical_controls", ["db_name"])
+    op.create_index("ix_clinical_controls_db_identifier", "clinical_controls", ["db_identifier"])
+    op.create_index("ix_clinical_controls_db_version", "clinical_controls", ["db_version"])
+
+    op.create_table(
+        "mapped_variants_clinical_controls",
+        sa.Column("mapped_variant_id", sa.Integer(), nullable=False),
+        sa.Column("clinical_control_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["mapped_variant_id"],
+            ["mapped_variants.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["clinical_control_id"],
+            ["clinical_controls.id"],
+        ),
+        sa.PrimaryKeyConstraint("mapped_variant_id", "clinical_control_id"),
+    )
+
+    # Convert any existing ClinVar variants into clinical control variants. Since
+    # this table is being update from a clinvar specific table, we assume all existing
+    # controls are from ClinVar.
+    op.execute(
+        """
+        INSERT INTO mapped_variants_clinical_controls (
+            mapped_variant_id,
+            clinical_control_id
+        )
+        SELECT id, clinvar_variant_id
+        FROM mapped_variants
+        WHERE clinvar_variant_id IS NOT NULL
+        """
+    )
+
+    op.execute("UPDATE clinical_controls SET db_name='ClinVar'")
+    op.alter_column("clinical_controls", "db_name", nullable=False)
+
+    op.drop_index("ix_mapped_variants_clinvar_variant_id", "mapped_variants")
+    op.drop_column("mapped_variants", "clinvar_variant_id")
+
+
+def downgrade():
+    op.rename_table("clinical_controls", "clinvar_variants")
+    op.execute("ALTER SEQUENCE clinical_controls_id_seq RENAME TO clinvar_variants_id_seq")
+    op.execute("ALTER INDEX clinical_controls_pkey RENAME TO clinvar_variants_pkey")
+
+    op.drop_index("ix_clinical_controls_gene_symbol", "clinical_controls")
+    op.drop_index("ix_clinical_controls_db_name", "clinical_controls")
+    op.drop_index("ix_clinical_controls_db_identifier", "clinical_controls")
+    op.drop_index("ix_clinical_controls_db_version", "clinical_controls")
+
+    op.alter_column("clinvar_variants", "db_version", nullable=False, new_column_name="clinvar_db_version")
+    op.alter_column("clinvar_variants", "db_identifier", nullable=False, new_column_name="allele_id")
+    op.drop_column("clinvar_variants", "db_name")
+
+    op.add_column(
+        "mapped_variants",
+        sa.Column("clinvar_variant_id", sa.Integer(), sa.ForeignKey("clinvar_variants.id"), nullable=True),
+    )
+
+    # Downgrades a many-to-many relationship to a one to many. This will result in data loss.
+    op.execute(
+        """
+        UPDATE mapped_variants
+        SET clinvar_variant_id=mapped_variants_clinical_controls.clinical_control_id
+        FROM mapped_variants_clinical_controls
+        WHERE mapped_variants_clinical_controls.mapped_variant_id=mapped_variants.id
+        """
+    )
+
+    op.create_index("ix_mapped_variants_clinvar_variant_id", "mapped_variants", ["clinvar_variant_id"])
+    op.drop_table("mapped_variants_clinical_controls")

alembic/versions/d6e5a9fde3c9_move_clingen_allele_id_to_mapped_.py

@@ -0,0 +1,53 @@
+"""move clingen allele id to mapped variants table
+
+Revision ID: d6e5a9fde3c9
+Revises: 695b73abe581
+Create Date: 2025-02-19 10:51:07.319962
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d6e5a9fde3c9"
+down_revision = "695b73abe581"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_variants_clingen_allele_id", table_name="variants")
+    op.add_column("mapped_variants", sa.Column("clingen_allele_id", sa.String(), nullable=True))
+    op.execute(
+        """
+        UPDATE mapped_variants
+        SET clingen_allele_id=variants.clingen_allele_id
+        FROM variants
+        WHERE variants.id=mapped_variants.variant_id
+        """
+    )
+    op.drop_column("variants", "clingen_allele_id")
+    op.create_index(
+        op.f("ix_mapped_variants_clingen_allele_id"), "mapped_variants", ["clingen_allele_id"], unique=False
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_mapped_variants_clingen_allele_id"), table_name="mapped_variants")
+    op.add_column("variants", sa.Column("clingen_allele_id", sa.String(), nullable=True))
+    op.execute(
+        """
+        UPDATE variants
+        SET clingen_allele_id=mapped_variants.clingen_allele_id
+        FROM mapped_variants
+        WHERE variants.id=mapped_variants.variant_id
+        """
+    )
+    op.drop_column("mapped_variants", "clingen_allele_id")
+    op.create_index("ix_variants_clingen_allele_id", "variants", ["clingen_allele_id"], unique=False)
+    # ### end Alembic commands ###

alembic/versions/e8a3b5d8f885_add_clingen_allele_ids.py

@@ -0,0 +1,27 @@
+"""Add ClinGen allele IDs
+
+Revision ID: e8a3b5d8f885
+Revises: 4726e4dddde8
+Create Date: 2025-01-27 18:55:09.283855
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "e8a3b5d8f885"
+down_revision = "4726e4dddde8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("variants", sa.Column("clingen_allele_id", sa.String(), nullable=True))
+    op.create_index(op.f("ix_variants_clingen_allele_id"), "variants", ["clingen_allele_id"], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f("ix_variants_clingen_allele_id"), table_name="variants")
+    op.drop_column("variants", "clingen_allele_id")

src/mavedb/constants.py

@@ -0,0 +1,4 @@
+import os
+
+MAVEDB_BASE_GIT = "https://github.com/VariantEffect/mavedb-api"
+MAVEDB_FRONTEND_URL = os.getenv("MAVE_FRONTEND_URL", "https://mavedb.org")

src/mavedb/lib/clingen/constants.py

@@ -0,0 +1,17 @@
+import os
+
+GENBOREE_ACCOUNT_NAME = os.getenv("GENBOREE_ACCOUNT_NAME")
+GENBOREE_ACCOUNT_PASSWORD = os.getenv("GENBOREE_ACCOUNT_PASSWORD")
+
+CLIN_GEN_TENANT = os.getenv("CLIN_GEN_TENANT")
+LDH_TENANT = os.getenv("LDH_TENANT")
+
+LDH_SUBMISSION_TYPE = "cg-ldh-ld-submission"
+LDH_ENTITY_NAME = "MaveDBMapping"
+LDH_ENTITY_ENDPOINT = "maveDb"  # for some reason, not the same :/
+
+DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100
+LDH_SUBMISSION_URL = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}"
+LDH_LINKED_DATA_URL = f"https://genboree.org/{LDH_TENANT}/{LDH_ENTITY_NAME}/id"
+
+LINKED_DATA_RETRY_THRESHOLD = 0.95

src/mavedb/lib/clingen/content_constructors.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+from uuid import uuid4
+
+from mavedb import __version__
+from mavedb.constants import MAVEDB_BASE_GIT, MAVEDB_FRONTEND_URL
+from mavedb.lib.types.clingen import LdhContentLinkedData, LdhContentSubject, LdhEvent, LdhSubmission
+from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE
+from mavedb.models.mapped_variant import MappedVariant
+from mavedb.models.variant import Variant
+
+
+def construct_ldh_submission_event(sbj: LdhContentSubject) -> LdhEvent:
+    return {
+        "type": LDH_SUBMISSION_TYPE,
+        "name": LDH_ENTITY_NAME,
+        "uuid": str(uuid4()),
+        "sbj": {"id": sbj["Variant"]["hgvs"], "type": "Variant", "format": "hgvs", "add": True},
+        "triggered": {
+            "by": {
+                "host": MAVEDB_BASE_GIT,
+                "id": "resource_published",
+                "iri": f"{MAVEDB_BASE_GIT}/releases/tag/v{__version__}",
+            },
+            "at": datetime.now().isoformat(),
+        },
+    }
+
+
+def construct_ldh_submission_subject(hgvs: str) -> LdhContentSubject:
+    return {"Variant": {"hgvs": hgvs}}
+
+
+def construct_ldh_submission_entity(variant: Variant, mapped_variant: MappedVariant) -> LdhContentLinkedData:
+    return {
+        # TODO#372: We try to make all possible fields that are non-nullable represented that way.
+        "MaveDBMapping": [
+            {
+                "entContent": {
+                    "mavedb_id": variant.urn,  # type: ignore
+                    "pre_mapped": mapped_variant.pre_mapped,  # type: ignore
+                    "post_mapped": mapped_variant.post_mapped,  # type: ignore
+                    "mapping_api_version": mapped_variant.mapping_api_version,  # type: ignore
+                    "score": variant.data["score_data"]["score"],  # type: ignore
+                },
+                "entId": variant.urn,  # type: ignore
+                "entIri": f"{MAVEDB_FRONTEND_URL}/{variant.urn}",  # type: ignore
+            }
+        ]
+    }
+
+
+def construct_ldh_submission(variant_content: list[tuple[str, Variant, MappedVariant]]) -> list[LdhSubmission]:
+    content_submission: list[LdhSubmission] = []
+    for hgvs, variant, mapped_variant in variant_content:
+        subject = construct_ldh_submission_subject(hgvs)
+        event = construct_ldh_submission_event(subject)
+        entity = construct_ldh_submission_entity(variant, mapped_variant)
+
+        content_submission.append(
+            {
+                "event": event,
+                "content": {"sbj": subject, "ld": entity},
+            }
+        )
+
+    return content_submission