Remove any variant type

CHANGELOG.md

@@ -12,6 +12,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 - Preselect MANE SELECT transcripts in the multi-step ClinVar variant add to submission process
 - Allow updating case with WTS Fraser and Outrider research files
 - Load research WTS outliers using the `scout load variants --outliers-research` command
+- Expand the command line to remove more types of variants. Now supports: `cancer`, `cancer_sv`, `fusion`, `mei`, `outlier`, `snv`, `str`, `sv`, `wts_outliers`
 ### Changed
 - Do not show overlapping gene panels badge on variants from cases runned without gene panels
 - Set case as research case if it contains any type of research variants
@@ -27,6 +28,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 - Don't parse SV frequencies for SNVs even if the name matches. Also accept "." as missing value for SV frequencies.
 - HPO search on WTS Outliers page
 
+
 ## [4.96]
 ### Added
 - Support case status assignment upon loading (by providing case status in the case config file)

scout/adapter/mongo/query.py

@@ -43,19 +43,13 @@ def build_case_query(
         return case_query
 
     def delete_variants_query(
-        self, case_id, variants_to_keep=[], min_rank_threshold=None, keep_ctg=[]
+        self,
+        case_id: str,
+        variants_to_keep: List[str] = [],
+        min_rank_threshold: Optional[int] = None,
+        keep_ctg: List[str] = [],
     ) -> dict:
-        """Build a query to delete variants from a case
-
-        Args:
-            case_id(str): id of a case
-            variants_to_keep(list): a list of variant ids
-            min_rank_threshold(int): remove variants with rank lower than this number
-            keep_ctg(list): exclude one of more variants categories from deletion. Example ["cancer", "cancer_sv"]
-
-        Return:
-            variant_query(dict): query dictionary
-        """
+        """Build a query to delete variants from a case (variant collection)."""
         variants_query = {}
         case_subquery = {"case_id": case_id}
 

scout/adapter/mongo/variant.py

@@ -877,11 +877,6 @@ def case_variants_count(self, case_id, institute_id, variant_type=None, force_up
                     }
                 }
         """
-        LOG.info(
-            "Retrieving variants by category for case: {0}, institute: {1}".format(
-                case_id, institute_id
-            )
-        )
 
         case_obj = self.case(case_id=case_id)
         variants_stats = case_obj.get("variants_stats") or {}

scout/commands/delete/delete_command.py

@@ -1,15 +1,16 @@
 import logging
+from typing import List, Tuple
 
 import click
 from flask import current_app, url_for
 from flask.cli import with_appcontext
 
-from scout.constants import CASE_STATUSES
+from scout.constants import ANALYSIS_TYPES, CASE_STATUSES, VARIANTS_TARGET_FROM_CATEGORY
 from scout.server.extensions import store
 
 LOG = logging.getLogger(__name__)
 
-BYTES_IN_ONE_GIGABYTE = 1073741824  # (1024*1024*1024)
+BYTES_IN_ONE_GIGABYTE = 1073741824
 DELETE_VARIANTS_HEADER = [
     "Case n.",
     "Ncases",
@@ -23,8 +24,18 @@
     "Total variants",
     "Removed variants",
 ]
+VARIANT_CATEGORIES = list(VARIANTS_TARGET_FROM_CATEGORY.keys()) + ["wts_outliers"]
 
-VARIANT_CATEGORIES = ["mei", "snv", "sv", "cancer", "cancer_sv", "str"]
+
+def _set_keep_ctg(keep_ctg: Tuple[str], rm_ctg: Tuple[str]) -> List[str]:
+    """Define the categories of variants that should not be removed."""
+    if keep_ctg and rm_ctg:
+        raise click.UsageError("Please use either '--keep-ctg' or '--rm-ctg' parameter, not both.")
+    if keep_ctg:
+        return list(keep_ctg)
+    if rm_ctg:
+        return list(set(VARIANT_CATEGORIES).difference(set(rm_ctg)))
+    return []
 
 
 @click.command("variants", short_help="Delete variants for one or more cases")
@@ -47,18 +58,25 @@
 @click.option("--older-than", type=click.INT, default=0, help="Older than (months)")
 @click.option(
     "--analysis-type",
-    type=click.Choice(["wgs", "wes", "panel"]),
+    type=click.Choice(ANALYSIS_TYPES),
     multiple=True,
     help="Type of analysis",
 )
 @click.option("--rank-threshold", type=click.INT, default=5, help="With rank threshold lower than")
 @click.option("--variants-threshold", type=click.INT, help="With more variants than")
+@click.option(
+    "--rm-ctg",
+    type=click.Choice(VARIANT_CATEGORIES),
+    multiple=True,
+    required=False,
+    help="Remove only the following categories",
+)
 @click.option(
     "--keep-ctg",
     type=click.Choice(VARIANT_CATEGORIES),
     multiple=True,
     required=False,
-    help="Do not delete one of more variant categories",
+    help="Keep the following categories",
 )
 @click.option(
     "--dry-run",
@@ -66,146 +84,164 @@
     help="Perform a simulation without removing any variant",
 )
 @with_appcontext
 def variants(
     user: str,
-    case_id: list,
+    case_id: tuple,
     case_file: str,
     institute: str,
-    status: list,
+    status: tuple,
     older_than: int,
-    analysis_type: list,
+    analysis_type: tuple,
     rank_threshold: int,
     variants_threshold: int,
-    keep_ctg: list,
+    rm_ctg: tuple,
+    keep_ctg: tuple,
     dry_run: bool,
 ) -> None:
     """Delete variants for one or more cases"""
 
     if case_file and case_id:
         click.echo(
             "You should specify either case ID (multiple times if needed) or the path to a text file containing a list of case IDs (one per line)."
         )
         return
 
     user_obj = store.user(user)
     if user_obj is None:
         click.echo(f"Could not find a user with email '{user}' in database")
         return
 
     total_deleted = 0
-    items_name = "deleted variants"
+
     if dry_run:
         click.echo("--------------- DRY RUN COMMAND ---------------")
-        items_name = "estimated deleted variants"
     else:
         click.confirm("Variants are going to be deleted from database. Continue?", abort=True)
 
     if case_file:
         case_id = [line.strip() for line in open(case_file).readlines() if line.strip()]
 
     case_query = store.build_case_query(
         case_ids=case_id,
         institute_id=institute,
         status=status,
         older_than=older_than,
         analysis_type=analysis_type,
     )
     # Estimate the average size of a variant document in database
     avg_var_size = store.collection_stats("variant").get("avgObjSize", 0)  # in bytes
 
     # Get all cases where case_query applies
     n_cases = store.case_collection.count_documents(case_query)
     cases = store.cases(query=case_query)
     filters = (
         f"Rank-score threshold:{rank_threshold}, case n. variants threshold:{variants_threshold}."
     )
     click.echo("\t".join(DELETE_VARIANTS_HEADER))
+    keep_ctg = _set_keep_ctg(keep_ctg=keep_ctg, rm_ctg=rm_ctg)
+
     for nr, case in enumerate(cases, 1):
         case_id = case["_id"]
         institute_id = case["owner"]
-        case_n_variants = store.variant_collection.count_documents({"case_id": case_id})
+        case_n_variants = store.variant_collection.count_documents(
+            {"case_id": case_id}
+        ) + store.omics_variant_collection.count_documents({"case_id": case_id})
         # Skip case if user provided a number of variants to keep and this number is less than total number of case variants
         if variants_threshold and case_n_variants < variants_threshold:
             continue
         # Get evaluated variants for the case that haven't been dismissed
         case_evaluated = store.evaluated_variants(case_id=case_id, institute_id=institute_id)
         evaluated_not_dismissed = [
             variant["_id"] for variant in case_evaluated if "dismiss_variant" not in variant
         ]
         # Do not remove variants that are either pinned, causative or evaluated not dismissed
         variants_to_keep = (
             case.get("suspects", []) + case.get("causatives", []) + evaluated_not_dismissed or []
         )
-        variants_query = store.delete_variants_query(
+
+        variants_query: dict = store.delete_variants_query(
             case_id, variants_to_keep, rank_threshold, keep_ctg
         )
 
         if dry_run:
+            items_name = "estimated deleted variants"
             # Just print how many variants would be removed for this case
             remove_n_variants = store.variant_collection.count_documents(variants_query)
-            total_deleted += remove_n_variants
+            remove_n_omics_variants = (
+                store.omics_variant_collection.count_documents(variants_query)
+                if "wts_outliers" not in keep_ctg
+                else 0
+            )
+            total_deleted += remove_n_variants + remove_n_omics_variants
             click.echo(
                 "\t".join(
                     [
                         str(nr),
                         str(n_cases),
                         case["owner"],
                         case["display_name"],
                         case_id,
                         case.get("track", ""),
                         str(case["analysis_date"]),
                         case.get("status", ""),
                         str(case.get("is_research", "")),
                         str(case_n_variants),
-                        str(remove_n_variants),
+                        str(remove_n_variants + remove_n_omics_variants),
                     ]
                 )
             )
             continue
 
         # delete variants specified by variants_query
-        result = store.variant_collection.delete_many(variants_query)
-        total_deleted += result.deleted_count
+        items_name = "deleted variants"
+        remove_n_variants = store.variant_collection.delete_many(variants_query).deleted_count
+        remove_n_omics_variants = (
+            store.omics_variant_collection.delete_many(variants_query).deleted_count
+            if "wts_outliers" not in keep_ctg
+            else 0
+        )
+
+        total_deleted += remove_n_variants + remove_n_omics_variants
         click.echo(
             "\t".join(
                 [
                     str(nr),
                     str(n_cases),
                     case["owner"],
                     case["display_name"],
                     case_id,
                     case.get("track", ""),
                     str(case["analysis_date"]),
                     case.get("status", ""),
                     str(case.get("is_research", "")),
                     str(case_n_variants),
-                    str(result.deleted_count),
+                    str(remove_n_variants + remove_n_omics_variants),
                 ]
             )
         )
 
         # Create event in database
         institute_obj = store.institute(case["owner"])
         with current_app.test_request_context("/cases"):
             url = url_for(
                 "cases.case",
                 institute_id=institute_obj["_id"],
                 case_name=case["display_name"],
             )
             store.remove_variants_event(
                 institute=institute_obj,
                 case=case,
                 user=user_obj,
                 link=url,
                 content=filters,
             )
 
         # Update case variants count
         store.case_variants_count(case_id, institute_obj["_id"], True)
 
     click.echo(f"Total {items_name}: {total_deleted}")
     click.echo(
         f"Estimated space freed (GB): {round((total_deleted * avg_var_size) / BYTES_IN_ONE_GIGABYTE, 4)}"
     )
 
 

scout/server/blueprints/cases/templates/cases/utils.html

@@ -291,7 +291,7 @@ <h5 class="modal-title" id="exampleModalLabel">Beacon submission</h5>
 
 {% macro pretty_variant(variant) %}
 
-  {% if variant.category in "str" %}
+  {% if variant.category and variant.category in "str" %}
         {{ variant.str_repid }} {{ variant.alternative | replace("<", " ") | replace(">", "")}}
 
   {% elif variant.category in ("snv", "cancer") %}

tests/commands/delete/test_delete_cmd.py

@@ -1,7 +1,4 @@
 # -*- coding: utf-8 -*-
-
-from pymongo import ASCENDING, IndexModel
-
 from scout.commands import cli
 from scout.server.extensions import store
 
@@ -62,7 +59,7 @@ def test_delete_variants(mock_app, case_obj, user_obj):
     )
     assert result.exit_code == 0
     n_initial_vars = sum(1 for _ in store.variant_collection.find())
-    n_variants_to_exclude = store.variant_collection.count_documents(VARIANTS_QUERY)
+    n_variants_to_delete = store.variant_collection.count_documents(VARIANTS_QUERY)
 
     # Then the function that delete variants should run without error
     cmd_params = [
@@ -88,7 +85,7 @@ def test_delete_variants(mock_app, case_obj, user_obj):
     # variants should be deleted
     n_current_vars = sum(1 for _ in store.variant_collection.find())
     assert n_current_vars < n_initial_vars
-    assert n_current_vars + n_variants_to_exclude == n_initial_vars
+    assert n_current_vars + n_variants_to_delete == n_initial_vars
     # and a relative event should be created
     event = store.event_collection.find_one({"verb": "remove_variants"})
     assert event["case"] == case_obj["_id"]
@@ -98,6 +95,43 @@ def test_delete_variants(mock_app, case_obj, user_obj):
     )
 
 
+def test_delete_omics_variants(mock_app, case_obj, user_obj):
+    """Test the delete variants command's ability to remove omics variants."""
+
+    # GIVEN a database with -omics variants
+    runner = mock_app.test_cli_runner()
+    result = runner.invoke(
+        cli, ["load", "variants", case_obj["_id"], "--outlier", "--rank-treshold", 0]
+    )
+    assert result.exit_code == 0
+    n_initial_vars = sum(1 for _ in store.variant_collection.find())
+    n_variants_to_delete = store.variant_collection.count_documents(VARIANTS_QUERY)
+    # WHEN variants are removed using the command line
+    cmd_params = [
+        "delete",
+        "variants",
+        "-u",
+        user_obj["email"],
+        "--rank-threshold",
+        0,
+        "--rm-ctg",
+        "wts_outliers",
+    ]
+    result = runner.invoke(cli, cmd_params, input="y")
+    assert result.exit_code == 0
+    assert "estimated deleted variants" not in result.output
+    assert "Estimated space freed" in result.output
+
+    # THEN the should all be gone
+    n_current_vars = sum(1 for _ in store.variant_collection.find())
+    assert n_current_vars == 0
+    assert n_current_vars + n_variants_to_delete == n_initial_vars
+    # and a relative event should be created
+    event = store.event_collection.find_one({"verb": "remove_variants"})
+    assert event["case"] == case_obj["_id"]
+    assert "Rank-score threshold:0" in event["content"]
+
+
 def test_delete_panel_non_existing(empty_mock_app, testpanel_obj):
     "Test the CLI command that deletes a gene panel"
     mock_app = empty_mock_app