diff --git a/CHANGELOG.md b/CHANGELOG.md index 479a0f910a..1b22183bec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Preselect MANE SELECT transcripts in the multi-step ClinVar variant add to submission process - Allow updating case with WTS Fraser and Outrider research files - Load research WTS outliers using the `scout load variants --outliers-research` command +- Expand the command line to remove more types of variants. Now supports: `cancer`, `cancer_sv`, `fusion`, `mei`, `outlier`, `snv`, `str`, `sv`, `wts_outliers` ### Changed - Do not show overlapping gene panels badge on variants from cases runned without gene panels - Set case as research case if it contains any type of research variants @@ -27,6 +28,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Don't parse SV frequencies for SNVs even if the name matches. Also accept "." as missing value for SV frequencies. - HPO search on WTS Outliers page + ## [4.96] ### Added - Support case status assignment upon loading (by providing case status in the case config file) diff --git a/scout/adapter/mongo/query.py b/scout/adapter/mongo/query.py index 06eccba396..284d052b75 100644 --- a/scout/adapter/mongo/query.py +++ b/scout/adapter/mongo/query.py @@ -43,19 +43,13 @@ def build_case_query( return case_query def delete_variants_query( - self, case_id, variants_to_keep=[], min_rank_threshold=None, keep_ctg=[] + self, + case_id: str, + variants_to_keep: List[str] = [], + min_rank_threshold: Optional[int] = None, + keep_ctg: List[str] = [], ) -> dict: - """Build a query to delete variants from a case - - Args: - case_id(str): id of a case - variants_to_keep(list): a list of variant ids - min_rank_threshold(int): remove variants with rank lower than this number - keep_ctg(list): exclude one of more variants categories from deletion. Example ["cancer", "cancer_sv"] - - Return: - variant_query(dict): query dictionary - """ + """Build a query to delete variants from a case (variant collection).""" variants_query = {} case_subquery = {"case_id": case_id} diff --git a/scout/adapter/mongo/variant.py b/scout/adapter/mongo/variant.py index 008e7bf6f3..aec18b4f9d 100644 --- a/scout/adapter/mongo/variant.py +++ b/scout/adapter/mongo/variant.py @@ -877,11 +877,6 @@ def case_variants_count(self, case_id, institute_id, variant_type=None, force_up } } """ - LOG.info( - "Retrieving variants by category for case: {0}, institute: {1}".format( - case_id, institute_id - ) - ) case_obj = self.case(case_id=case_id) variants_stats = case_obj.get("variants_stats") or {} diff --git a/scout/commands/delete/delete_command.py b/scout/commands/delete/delete_command.py index 1d11d69d91..59cd1f612c 100644 --- a/scout/commands/delete/delete_command.py +++ b/scout/commands/delete/delete_command.py @@ -1,15 +1,16 @@ import logging +from typing import List, Tuple import click from flask import current_app, url_for from flask.cli import with_appcontext -from scout.constants import CASE_STATUSES +from scout.constants import ANALYSIS_TYPES, CASE_STATUSES, VARIANTS_TARGET_FROM_CATEGORY from scout.server.extensions import store LOG = logging.getLogger(__name__) -BYTES_IN_ONE_GIGABYTE = 1073741824 # (1024*1024*1024) +BYTES_IN_ONE_GIGABYTE = 1073741824 DELETE_VARIANTS_HEADER = [ "Case n.", "Ncases", @@ -23,8 +24,18 @@ "Total variants", "Removed variants", ] +VARIANT_CATEGORIES = list(VARIANTS_TARGET_FROM_CATEGORY.keys()) + ["wts_outliers"] -VARIANT_CATEGORIES = ["mei", "snv", "sv", "cancer", "cancer_sv", "str"] + +def _set_keep_ctg(keep_ctg: Tuple[str], rm_ctg: Tuple[str]) -> List[str]: + """Define the categories of variants that should not be removed.""" + if keep_ctg and rm_ctg: + raise click.UsageError("Please use either '--keep-ctg' or '--rm-ctg' parameter, not both.") + if keep_ctg: + return list(keep_ctg) + if rm_ctg: + return list(set(VARIANT_CATEGORIES).difference(set(rm_ctg))) + return [] @click.command("variants", short_help="Delete variants for one or more cases") @@ -47,18 +58,25 @@ @click.option("--older-than", type=click.INT, default=0, help="Older than (months)") @click.option( "--analysis-type", - type=click.Choice(["wgs", "wes", "panel"]), + type=click.Choice(ANALYSIS_TYPES), multiple=True, help="Type of analysis", ) @click.option("--rank-threshold", type=click.INT, default=5, help="With rank threshold lower than") @click.option("--variants-threshold", type=click.INT, help="With more variants than") +@click.option( + "--rm-ctg", + type=click.Choice(VARIANT_CATEGORIES), + multiple=True, + required=False, + help="Remove only the following categories", +) @click.option( "--keep-ctg", type=click.Choice(VARIANT_CATEGORIES), multiple=True, required=False, - help="Do not delete one of more variant categories", + help="Keep the following categories", ) @click.option( "--dry-run", @@ -68,15 +86,16 @@ @with_appcontext def variants( user: str, - case_id: list, + case_id: tuple, case_file: str, institute: str, - status: list, + status: tuple, older_than: int, - analysis_type: list, + analysis_type: tuple, rank_threshold: int, variants_threshold: int, - keep_ctg: list, + rm_ctg: tuple, + keep_ctg: tuple, dry_run: bool, ) -> None: """Delete variants for one or more cases""" @@ -93,10 +112,9 @@ def variants( return total_deleted = 0 - items_name = "deleted variants" + if dry_run: click.echo("--------------- DRY RUN COMMAND ---------------") - items_name = "estimated deleted variants" else: click.confirm("Variants are going to be deleted from database. Continue?", abort=True) @@ -120,10 +138,14 @@ def variants( f"Rank-score threshold:{rank_threshold}, case n. variants threshold:{variants_threshold}." ) click.echo("\t".join(DELETE_VARIANTS_HEADER)) + keep_ctg = _set_keep_ctg(keep_ctg=keep_ctg, rm_ctg=rm_ctg) + for nr, case in enumerate(cases, 1): case_id = case["_id"] institute_id = case["owner"] - case_n_variants = store.variant_collection.count_documents({"case_id": case_id}) + case_n_variants = store.variant_collection.count_documents( + {"case_id": case_id} + ) + store.omics_variant_collection.count_documents({"case_id": case_id}) # Skip case if user provided a number of variants to keep and this number is less than total number of case variants if variants_threshold and case_n_variants < variants_threshold: continue @@ -136,14 +158,21 @@ def variants( variants_to_keep = ( case.get("suspects", []) + case.get("causatives", []) + evaluated_not_dismissed or [] ) - variants_query = store.delete_variants_query( + + variants_query: dict = store.delete_variants_query( case_id, variants_to_keep, rank_threshold, keep_ctg ) if dry_run: + items_name = "estimated deleted variants" # Just print how many variants would be removed for this case remove_n_variants = store.variant_collection.count_documents(variants_query) - total_deleted += remove_n_variants + remove_n_omics_variants = ( + store.omics_variant_collection.count_documents(variants_query) + if "wts_outliers" not in keep_ctg + else 0 + ) + total_deleted += remove_n_variants + remove_n_omics_variants click.echo( "\t".join( [ @@ -157,15 +186,22 @@ def variants( case.get("status", ""), str(case.get("is_research", "")), str(case_n_variants), - str(remove_n_variants), + str(remove_n_variants + remove_n_omics_variants), ] ) ) continue # delete variants specified by variants_query - result = store.variant_collection.delete_many(variants_query) - total_deleted += result.deleted_count + items_name = "deleted variants" + remove_n_variants = store.variant_collection.delete_many(variants_query).deleted_count + remove_n_omics_variants = ( + store.omics_variant_collection.delete_many(variants_query).deleted_count + if "wts_outliers" not in keep_ctg + else 0 + ) + + total_deleted += remove_n_variants + remove_n_omics_variants click.echo( "\t".join( [ @@ -179,7 +215,7 @@ def variants( case.get("status", ""), str(case.get("is_research", "")), str(case_n_variants), - str(result.deleted_count), + str(remove_n_variants + remove_n_omics_variants), ] ) ) diff --git a/scout/server/blueprints/cases/templates/cases/utils.html b/scout/server/blueprints/cases/templates/cases/utils.html index ec95f39ad0..90644a57e6 100644 --- a/scout/server/blueprints/cases/templates/cases/utils.html +++ b/scout/server/blueprints/cases/templates/cases/utils.html @@ -291,7 +291,7 @@ {% macro pretty_variant(variant) %} - {% if variant.category in "str" %} + {% if variant.category and variant.category in "str" %} {{ variant.str_repid }} {{ variant.alternative | replace("<", " ") | replace(">", "")}} {% elif variant.category in ("snv", "cancer") %} diff --git a/tests/commands/delete/test_delete_cmd.py b/tests/commands/delete/test_delete_cmd.py index 61af81a753..448412af5c 100644 --- a/tests/commands/delete/test_delete_cmd.py +++ b/tests/commands/delete/test_delete_cmd.py @@ -1,7 +1,4 @@ # -*- coding: utf-8 -*- - -from pymongo import ASCENDING, IndexModel - from scout.commands import cli from scout.server.extensions import store @@ -62,7 +59,7 @@ def test_delete_variants(mock_app, case_obj, user_obj): ) assert result.exit_code == 0 n_initial_vars = sum(1 for _ in store.variant_collection.find()) - n_variants_to_exclude = store.variant_collection.count_documents(VARIANTS_QUERY) + n_variants_to_delete = store.variant_collection.count_documents(VARIANTS_QUERY) # Then the function that delete variants should run without error cmd_params = [ @@ -88,7 +85,7 @@ def test_delete_variants(mock_app, case_obj, user_obj): # variants should be deleted n_current_vars = sum(1 for _ in store.variant_collection.find()) assert n_current_vars < n_initial_vars - assert n_current_vars + n_variants_to_exclude == n_initial_vars + assert n_current_vars + n_variants_to_delete == n_initial_vars # and a relative event should be created event = store.event_collection.find_one({"verb": "remove_variants"}) assert event["case"] == case_obj["_id"] @@ -98,6 +95,43 @@ def test_delete_variants(mock_app, case_obj, user_obj): ) +def test_delete_omics_variants(mock_app, case_obj, user_obj): + """Test the delete variants command's ability to remove omics variants.""" + + # GIVEN a database with -omics variants + runner = mock_app.test_cli_runner() + result = runner.invoke( + cli, ["load", "variants", case_obj["_id"], "--outlier", "--rank-treshold", 0] + ) + assert result.exit_code == 0 + n_initial_vars = sum(1 for _ in store.variant_collection.find()) + n_variants_to_delete = store.variant_collection.count_documents(VARIANTS_QUERY) + # WHEN variants are removed using the command line + cmd_params = [ + "delete", + "variants", + "-u", + user_obj["email"], + "--rank-threshold", + 0, + "--rm-ctg", + "wts_outliers", + ] + result = runner.invoke(cli, cmd_params, input="y") + assert result.exit_code == 0 + assert "estimated deleted variants" not in result.output + assert "Estimated space freed" in result.output + + # THEN the should all be gone + n_current_vars = sum(1 for _ in store.variant_collection.find()) + assert n_current_vars == 0 + assert n_current_vars + n_variants_to_delete == n_initial_vars + # and a relative event should be created + event = store.event_collection.find_one({"verb": "remove_variants"}) + assert event["case"] == case_obj["_id"] + assert "Rank-score threshold:0" in event["content"] + + def test_delete_panel_non_existing(empty_mock_app, testpanel_obj): "Test the CLI command that deletes a gene panel" mock_app = empty_mock_app