Annotation rendering configuration new section updates

backend/tests/integration/test_analysis_routers.py

@@ -87,7 +87,7 @@ def test_import_analysis_with_phenotips_json(
 
             phenotips_file.close()
 
-            assert mock_annotation_queue.put.call_count == 29
+            assert mock_annotation_queue.put.call_count == 45
 
             mock_background_add_task.assert_called_once_with(
                 AnnotationService.process_tasks, mock_annotation_queue, mock_repositories['genomic_unit']

backend/tests/unit/core/test_annotate.py

@@ -11,7 +11,7 @@ def test_queuing_annotations_for_genomic_units(cpam0046_analysis, annotation_col
     annotation_service = AnnotationService(annotation_collection)
     mock_queue = Mock()
     annotation_service.queue_annotation_tasks(cpam0046_analysis, mock_queue)
-    assert mock_queue.put.call_count == 29
+    assert mock_queue.put.call_count == 45
 
 
 # Patching the temporary helper method that is writing to a file, this will be
@@ -52,11 +52,11 @@ def dependency_mock_side_effect(*args, **kwargs):  # pylint: disable=unused-argu
     AnnotationService.process_tasks(cpam0046_annotation_queue, mock_genomic_unit_collection)
     assert cpam0046_annotation_queue.empty()
 
-    assert http_task_annotate.call_count == 23
+    assert http_task_annotate.call_count == 35
     assert none_task_annotate.call_count == 0
-    assert forge_task_annotate.call_count == 6
+    assert forge_task_annotate.call_count == 10
 
-    assert annotate_extract_mock.call_count == 29
+    assert annotate_extract_mock.call_count == 45
 
 
 @patch("src.core.annotation.log_to_file")
@@ -86,11 +86,11 @@ def test_processing_cpam0002_annotations_tasks(
 
     AnnotationService.process_tasks(cpam0002_annotation_queue, mock_genomic_unit_collection)
 
-    assert http_task_annotate.call_count == 23
-    assert forge_task_annotate.call_count == 6
+    assert http_task_annotate.call_count == 35
+    assert forge_task_annotate.call_count == 10
     assert none_task_annotate.call_count == 0
 
-    assert annotate_extract_mock.call_count == 29
+    assert annotate_extract_mock.call_count == 45
 
     mock_genomic_unit_collection.annotate_genomic_unit.assert_called()
 

backend/tests/unit/repository/test_annotation_collection.py

@@ -8,13 +8,13 @@ def test_get_datasets_configuration_by_type(annotation_collection):
     """Tests getting the datasets for the provided types of genomic units"""
     types = set({GenomicUnitType.GENE, GenomicUnitType.HGVS_VARIANT})
     datasets = annotation_collection.datasets_to_annotate_by_type(types)
-    assert len(datasets) == 29
+    assert len(datasets) == 45
 
 
 def test_get_datasets_to_annotate_for_units(annotation_collection, genomic_units_for_annotation):
     """Tests if the configuration for datasets is return as expected"""
     actual_configuration = annotation_collection.datasets_to_annotate_for_units(genomic_units_for_annotation)
-    assert len(actual_configuration["gene"]) == 19
+    assert len(actual_configuration["gene"]) == 35
     assert len(actual_configuration["hgvs_variant"]) == 10
 
 

backend/tests/unit/repository/test_genomic_unit_collection.py

@@ -14,7 +14,7 @@
 def test_find_genomic_units(genomic_unit_collection):
     """ Gets all the genomic units from the genomic unit collection """
     all_genomic_units = genomic_unit_collection.all()
-    assert len(all_genomic_units) == 14
+    assert len(all_genomic_units) == 18
 
 
 def test_transcript_annotation_not_exist_with_no_annotations(genomic_unit_collection, hgvs_variant_genomic_unit_json):

etc/api/annotate-all-existing-analyses.sh

@@ -0,0 +1,77 @@
+#! /bin/bash
+# ./annotate-all-existing-analyses.sh
+
+usage() {
+  echo " "
+  echo "usage: $0"
+  echo " "
+  echo " -h Prints usage"
+  echo " "
+  echo "Kicks off annotation jobs for all the existing analyses in Rosalution"
+  echo " "
+  echo "To run the annotations, please log in to Rosalution to retrieve the"
+  echo "Client ID and Client Secret credentials. These can be found by"
+  echo "clicking on your username or going to: <rosalution url>/rosalution/account"
+  echo " "
+  echo "Note: This script may need to be run multiple times due to the Rosalution"
+  echo "annotation system not being built for large sets of annotations queued."
+  echo " "
+  echo "Please install jq for this script to work. https://stedolan.github.io/jq/"
+  echo " "
+  exit
+}
+
+while getopts ":h" opt; do
+  case $opt in
+    h) usage;;
+    \?) echo "Invalid option -$OPTARG" && exit 127;;
+  esac
+done
+
+if ! jq --version &> /dev/null
+then
+    echo "Error: jq could not be found. Exiting."
+    usage
+fi
+
+echo "Please enter your Client Id";
+read -r CLIENT_ID;
+
+echo "Please enter your Client Secret";
+read -r -s CLIENT_SECRET;
+
+if [ -z "${CLIENT_ID}" ] || [ -z "${CLIENT_SECRET}" ]; then
+  echo " "
+  echo "Please enter required credentials."
+  usage
+fi
+
+echo "Fetching valid authentication token..."
+
+AUTH_TOKEN=$(curl -s -X 'POST' \
+  "http://local.rosalution.cgds/rosalution/api/auth/token" \
+  -H "accept: application/json" \
+  -H "Content-Type: application/x-www-form-urlencoded" \
+  -d "grant_type=&scope=&client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET" | jq -r '.access_token')
+
+echo "Fetching existing analyses in Rosalution..."
+
+RESPONSE=$(curl -s -X "GET" \
+  "http://local.rosalution.cgds/rosalution/api/analysis/" \
+  -H "accept: application/json" \
+  -H "Authorization: Bearer $AUTH_TOKEN")
+
+ANALYSES=()
+while IFS='' read -r line; do ANALYSES+=("$line"); done < <(echo "$RESPONSE" | jq -c '[.[].name]')
+
+echo "${ANALYSES[@]}" | jq -r '.[]' | while read -r ANALYSIS; do
+  echo "Starting annotations for analysis $ANALYSIS..."
+  curl -s -X "POST" \
+      "http://local.rosalution.cgds/rosalution/api/annotate/$ANALYSIS" \
+      -H "accept: application/json" \
+      -H "Authorization: Bearer $AUTH_TOKEN" \
+      > /dev/null
+  sleep 5
+done
+
+echo "Done. Exiting."

etc/fixtures/initial-seed/annotations-config.json

@@ -111,37 +111,73 @@
     "attribute": "{ \"HPO_gene_search_url\": .HPO_gene_search_url }"
   },
   {
-    "data_set": "Model Systems - Rat",
-    "data_source": "Aliance Genome",
+    "data_set": "Rat Gene Identifier",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/search_autocomplete?q={gene}",
+    "attribute": ".results[] | { \"name_key\": .name_key, \"name\": .name, \"primaryKey\": .primaryKey, \"searchKey\": \"{gene} (Rno)\" } | . + { \"searchKey\": .searchKey | ascii_downcase, \"name_key\": .name_key | ascii_downcase } |  select( .name_key == .searchKey) | { \"Rat Gene Identifier\": .primaryKey }"
+  },
+  {
+    "data_set": "Rat_Alliance_Genome_Automated_Summary",
+    "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/gene/{Rat Gene Identifier}",
-    "attribute": "{ \"Model Systems - Rat\": .geneSynopsis}",
+    "attribute": "{ \"Rat_Alliance_Genome_Automated_Summary\": .automatedGeneSynopsis }",
     "dependencies": ["Rat Gene Identifier"]
   },
   {
-    "data_set": "Rat Gene Identifier",
+    "data_set": "Rat_Alliance_Genome_RGD_Summary",
     "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
-    "url": "https://www.alliancegenome.org/api/search_autocomplete?q={gene}",
-    "attribute": ".results[] | select( .primaryKey | contains(\"RGD\")) | { \"Rat Gene Identifier\": .primaryKey }"
+    "url": "https://www.alliancegenome.org/api/gene/{Rat Gene Identifier}",
+    "attribute": "{ \"Rat_Alliance_Genome_RGD_Summary\": .geneSynopsis}",
+    "dependencies": ["Rat Gene Identifier"]
+  },
+  {
+    "data_set": "Rat_Alliance_Genome_Models",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{Rat Gene Identifier}/models?limit=100",
+    "attribute": "{ \"Rat_Alliance_Genome_Models\": [.results[]] }",
+    "dependencies": ["Rat Gene Identifier"]
   },
   {
     "data_set": "Mouse Gene Identifier",
     "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/search_autocomplete?q={gene}",
-    "attribute": ".results[] | select( .primaryKey | contains(\"MGI\")) | select( .name | contains(\"-\") | not ) |{ \"Mouse Gene Identifier\": .primaryKey }"
+    "attribute": ".results[] | { \"name_key\": .name_key, \"name\": .name, \"primaryKey\": .primaryKey, \"searchKey\": \"{gene} (Mmu)\" } | . + { \"searchKey\": .searchKey | ascii_downcase, \"name_key\": .name_key | ascii_downcase } |  select( .name_key == .searchKey) | { \"Mouse Gene Identifier\": .primaryKey }"
   },
   {
-    "data_set": "Model Systems - Mouse - Automated",
-    "data_source": "Aliance Genome",
+    "data_set": "Mouse_Alliance_Genome_Automated_Summary",
+    "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/gene/{Mouse Gene Identifier}",
-    "attribute": "{ \"Model Systems - Mouse\": .automatedGeneSynopsis}",
+    "attribute": "{ \"Mouse_Alliance_Genome_Automated_Summary\": .automatedGeneSynopsis }",
+    "dependencies": ["Mouse Gene Identifier"]
+  },
+  {
+    "data_set": "Mouse_Alliance_Genome_MGI_Summary",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{Mouse Gene Identifier}",
+    "attribute": "{ \"Mouse_Alliance_Genome_MGI_Summary\": .geneSynopsis }",
+    "dependencies": ["Mouse Gene Identifier"]
+  },
+  {
+    "data_set": "Mouse_Alliance_Genome_Models",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{Mouse Gene Identifier}/models?limit=100",
+    "attribute": "{ \"Mouse_Alliance_Genome_Models\": [.results[]] }",
     "dependencies": ["Mouse Gene Identifier"]
   },
   {
@@ -150,15 +186,33 @@
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/search_autocomplete?q={gene}",
-    "attribute": ".results[] | select( .primaryKey | contains(\"ZFIN\")) | select( .name | contains(\"-\") | not ) |{ \"Zebrafish Gene Identifier\": .primaryKey }"
+    "attribute": ".results[] | { \"name_key\": .name_key, \"name\": .name, \"primaryKey\": .primaryKey, \"searchKey\": \"{gene} (Dre)\" } | . + { \"searchKey\": .searchKey | ascii_downcase, \"name_key\": .name_key | ascii_downcase } |  select( .name_key == .searchKey) | { \"Zebrafish Gene Identifier\": .primaryKey }"
   },
   {
-    "data_set": "Model Systems - Zebrafish - Automated",
+    "data_set": "Zebrafish_Alliance_Genome_Automated_Summary",
     "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/gene/{Zebrafish Gene Identifier}",
-    "attribute": "{ \"Model Systems - Zebrafish\": .automatedGeneSynopsis}",
+    "attribute": "{ \"Zebrafish_Alliance_Genome_Automated_Summary\": .automatedGeneSynopsis }",
+    "dependencies": ["Zebrafish Gene Identifier"]
+  },
+  {
+    "data_set": "Zebrafish_Alliance_Genome_ZFIN_Summary",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{Zebrafish Gene Identifier}",
+    "attribute": "{ \"Zebrafish_Alliance_Genome_ZFIN_Summary\": .geneSynopsis }",
+    "dependencies": ["Zebrafish Gene Identifier"]
+  },
+  {
+    "data_set": "Zebrafish_Alliance_Genome_Models",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{Zebrafish Gene Identifier}/models?limit=100",
+    "attribute": "{ \"Zebrafish_Alliance_Genome_Models\": [.results[]] }",
     "dependencies": ["Zebrafish Gene Identifier"]
   },
   {
@@ -167,15 +221,33 @@
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/search_autocomplete?q={gene}",
-    "attribute": ".results[] | select( .primaryKey | contains(\"WB\")) | select( .name | contains(\"-\") | not ) |{ \"C-Elegens Gene Identifier\": .primaryKey }"
+    "attribute": ".results[] | { \"name_key\": .name_key, \"name\": .name, \"primaryKey\": .primaryKey, \"searchKey\": \"{gene} (Cel)\" } | . + { \"searchKey\": .searchKey | ascii_downcase, \"name_key\": .name_key | ascii_downcase } |  select( .name_key == .searchKey) | { \"C-Elegens Gene Identifier\": .primaryKey }"
   },
   {
-    "data_set": "Model Systems - C-Elegens - Automated",
+    "data_set": "C-Elegens_Alliance_Genome_Automated_Summary",
     "data_source": "Alliance Genome",
     "genomic_unit_type": "gene",
     "annotation_source_type": "http",
     "url": "https://www.alliancegenome.org/api/gene/{C-Elegens Gene Identifier}",
-    "attribute": "{ \"Model Systems - C-Elegens\": .automatedGeneSynopsis}",
+    "attribute": "{ \"C-Elegens_Alliance_Genome_Automated_Summary\": .automatedGeneSynopsis }",
+    "dependencies": ["C-Elegens Gene Identifier"]
+  },
+  {
+    "data_set": "C-Elegens_Alliance_Genome_WB_Summary",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{C-Elegens Gene Identifier}",
+    "attribute": "{ \"C-Elegens_Alliance_Genome_WB_Summary\": .geneSynopsis }",
+    "dependencies": ["C-Elegens Gene Identifier"]
+  },
+  {
+    "data_set": "C-Elegens_Alliance_Genome_Models",
+    "data_source": "Alliance Genome",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "http",
+    "url": "https://www.alliancegenome.org/api/gene/{C-Elegens Gene Identifier}/models?limit=100",
+    "attribute": "{ \"C-Elegens_Alliance_Genome_Models\": [.results[]] }",
     "dependencies": ["C-Elegens Gene Identifier"]
   },
   {
@@ -249,5 +321,77 @@
     "annotation_source_type": "http",
     "url": "http://grch37.rest.ensembl.org/vep/human/hgvs/{hgvs_variant}?content-type=application/json;refseq=1;",
     "attribute": ".[].transcript_consequences[] | { impact: .impact, transcript_id: .transcript_id }"
+  },
+  {
+    "data_set": "Rat_Alliance_Genome_url",
+    "data_source": "Rosalution",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "forge",
+    "base_string": "https://www.alliancegenome.org/gene/{Rat Gene Identifier}",
+    "attribute": "{ \"Rat_Alliance_Genome_url\": .Rat_Alliance_Genome_url }",
+    "dependencies": ["Rat Gene Identifier"]
+  },
+  {
+    "data_set": "Rat_Rat_Genome_Database_url",
+    "data_source": "Alliance genome",
+    "annotation_source_type": "http",
+    "genomic_unit_type": "gene",
+    "url": "https://www.alliancegenome.org/api/gene/{Rat Gene Identifier}",
+    "attribute": "{ \"Rat_Rat_Genome_Database_url\": .modCrossRefCompleteUrl }",
+    "dependencies": ["Rat Gene Identifier"]
+  },
+  {
+    "data_set": "Mouse_Alliance_Genome_url",
+    "data_source": "Rosalution",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "forge",
+    "base_string": "https://www.alliancegenome.org/gene/{Mouse Gene Identifier}",
+    "attribute": "{ \"Mouse_Alliance_Genome_url\": .Mouse_Alliance_Genome_url }",
+    "dependencies": ["Mouse Gene Identifier"]
+  },
+  {
+    "data_set": "Mouse_Mouse_Genome_Database_url",
+    "data_source": "Alliance genome",
+    "annotation_source_type": "http",
+    "genomic_unit_type": "gene",
+    "url": "https://www.alliancegenome.org/api/gene/{Mouse Gene Identifier}",
+    "attribute": "{ \"Mouse_Mouse_Genome_Database_url\": .modCrossRefCompleteUrl }",
+    "dependencies": ["Mouse Gene Identifier"]
+  },
+  {
+    "data_set": "Zebrafish_Alliance_Genome_url",
+    "data_source": "Rosalution",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "forge",
+    "base_string": "https://www.alliancegenome.org/gene/{Zebrafish Gene Identifier}",
+    "attribute": "{ \"Zebrafish_Alliance_Genome_url\": .Zebrafish_Alliance_Genome_url }",
+    "dependencies": ["Zebrafish Gene Identifier"]
+  },
+  {
+    "data_set": "Zebrafish_Zebrafish_Information_Network_url",
+    "data_source": "Alliance genome",
+    "annotation_source_type": "http",
+    "genomic_unit_type": "gene",
+    "url": "https://www.alliancegenome.org/api/gene/{Zebrafish Gene Identifier}",
+    "attribute": "{ \"Zebrafish_Zebrafish_Information_Network_url\": .modCrossRefCompleteUrl }",
+    "dependencies": ["Zebrafish Gene Identifier"]
+  },
+  {
+    "data_set": "C-Elegens_Alliance_Genome_url",
+    "data_source": "Rosalution",
+    "genomic_unit_type": "gene",
+    "annotation_source_type": "forge",
+    "base_string": "https://www.alliancegenome.org/gene/{C-Elegens Gene Identifier}",
+    "attribute": "{ \"C-Elegens_Alliance_Genome_url\": .C-Elegens_Alliance_Genome_url }",
+    "dependencies": ["C-Elegens Gene Identifier"]
+  },
+  {
+    "data_set": "C-Elegens_Worm_Base_url",
+    "data_source": "Alliance genome",
+    "annotation_source_type": "http",
+    "genomic_unit_type": "gene",
+    "url": "https://www.alliancegenome.org/api/gene/{C-Elegens Gene Identifier}",
+    "attribute": "{ \"C-Elegens_Worm_Base_url\": .modCrossRefCompleteUrl }",
+    "dependencies": ["C-Elegens Gene Identifier"]
   }
 ]