TheAnvil Release Q3 1598473585

gen3.theanvil.io/etlMapping.yaml

@@ -1,119 +1,121 @@
 mappings:
-- name: gen3.theanvil.io_subject
-  doc_type: subject
-  type: aggregator
-  root: subject
-  props:
-  - name: submitter_id
-  - name: project_id
-  - name: sex
-  - name: age_value
-  - name: ancestry
-  - name: disease_description
-  - name: phenotype_present
-  - name: phenotype_absent
-  - name: disease_id
-  - name: solve_state
-  - name: congenital_status
-  - name: age_of_onset
-  aggregated_props:
-  - name: sample_provider
-    path: samples
-    src: sample_provider
-    fn: set
-  - name: tissue_affected_status
-    path: samples
-    src: tissue_affected_status
-    fn: set
-  - name: tissue_type
-    path: samples
-    src: tissue_type
-    fn: set
-  - name: sample_type
-    path: samples
-    src: sample_type
-    fn: set
-  - name: original_material_type
-    path: samples
-    src: original_material_type
-    fn: set
-  - name: exome_capture_platform
-    path: samples.sequencings
-    src: exome_capture_platform
-    fn: set
-  - name: library_prep_kit_method
-    path: samples.sequencings
-    src: library_prep_kit_method
-    fn: set
-  - name: capture_region_bed_file
-    path: samples.sequencings
-    src: capture_region_bed_file
-    fn: set
-  - name: reference_genome_build
-    path: samples.sequencings
-    src: reference_genome_build
-    fn: set
-  - name: sequencing_assay
-    path: samples.sequencings
-    src: sequencing_assay
-    fn: set
-  - name: alignment_method
-    path: samples.sequencings
-    src: alignment_method
-    fn: set
-  - name: data_processing_pipeline
-    path: samples.sequencings
-    src: data_processing_pipeline
-    fn: set
-  - name: _samples_count
-    path: samples
-    fn: count
-  - name: _sequencings_count
-    path: samples.sequencings
-    fn: count
-  joining_props:
-  - index: file
-    join_on: subject_id
+  - name: gen3.theanvil.io_subject
+    doc_type: subject
+    type: aggregator
+    root: subject
     props:
-    - name: data_format
-      src: data_format
-      fn: set
-    - name: data_type
-      src: data_type
-      fn: set
-  parent_props:
-  - path: projects[project_code:code]
-  - path: projects[project_name:name]
-  - path: projects[project_dbgap_phs:dbgap_phs]
-  - path: projects[project_dbgap_consent_text:dbgap_consent_text]
-  - path: projects[project_short_name:short_name]
-  - path: projects[project_dbgap_accession_number:dbgap_accession_number]
-- name: gen3.theanvil.io_file
-  doc_type: file
-  type: collector
-  root: None
-  category: data_file
-  props:
-  - name: project_id
-  - name: submitter_id
-  - name: object_id
-  - name: md5sum
-  - name: file_name
-  - name: file_size
-  - name: data_format
-  - name: data_type
-  - name: state
-  - name: data_category
-  - name: analyte_type
-  - name: sequencing_assay
-  injecting_props:
-    subject:
-      props:
-      - name: subject_id
-        src: id
+      - name: submitter_id
+      - name: project_id
+      - name: sex
+      - name: age_value
+      - name: ancestry
+      - name: disease_description
+      - name: phenotype_present
+      - name: phenotype_absent
+      - name: disease_id
+      - name: solve_state
+      - name: congenital_status
+      - name: age_of_onset
+#    flatten_props:
+    aggregated_props:
+      - name: sample_provider
+        path: samples
+        src: sample_provider
         fn: set
-      - name: subject_submitter_id
-        src: submitter_id
+      - name: tissue_affected_status
+        path: samples
+        src: tissue_affected_status
         fn: set
-      - name: project_id
+      - name: tissue_type
+        path: samples
+        src: tissue_type
+        fn: set
+      - name: sample_type
+        path: samples
+        src: sample_type
+        fn: set
+      - name: original_material_type
+        path: samples
+        src: original_material_type
+        fn: set
+      - name: exome_capture_platform
+        path: samples.sequencings
+        src: exome_capture_platform
+        fn: set
+      - name: library_prep_kit_method
+        path: samples.sequencings
+        src: library_prep_kit_method
+        fn: set
+      - name: capture_region_bed_file
+        path: samples.sequencings
+        src: capture_region_bed_file
         fn: set
+      - name: reference_genome_build
+        path: samples.sequencings
+        src: reference_genome_build
+        fn: set
+      - name: sequencing_assay
+        path: samples.sequencings
+        src: sequencing_assay
+        fn: set
+      - name: alignment_method
+        path: samples.sequencings
+        src: alignment_method
+        fn: set
+      - name: data_processing_pipeline
+        path: samples.sequencings
+        src: data_processing_pipeline
+        fn: set
+      - name: _samples_count
+        path: samples
+        fn: count
+      - name: _sequencings_count
+        path: samples.sequencings
+        fn: count
+    joining_props:
+      - index: file
+        join_on: subject_id
+        props:
+          - name: data_format
+            src: data_format
+            fn: set
+          - name: data_type
+            src: data_type
+            fn: set
+    parent_props:
+      - path: projects[project_code:code]
+      - path: projects[project_name:name]
+      - path: projects[project_dbgap_phs:dbgap_phs]
+      - path: projects[project_dbgap_consent_text:dbgap_consent_text]
+      - path: projects[project_short_name:short_name]
+      - path: projects[project_dbgap_accession_number:dbgap_accession_number]
+
+  - name: gen3.theanvil.io_file
+    doc_type: file
+    type: collector
+    root: None
+    category: data_file
+    props:
+      - name: project_id
+      - name: submitter_id
+      - name: object_id
+      - name: md5sum
+      - name: file_name
+      - name: file_size
+      - name: data_format
+      - name: data_type
+      - name: state
+      - name: data_category
+      - name: analyte_type
+      - name: sequencing_assay      
+    injecting_props:
+      subject:
+        props:
+          - name: subject_id
+            src: id
+            fn: set
+          - name: subject_submitter_id
+            src: submitter_id
+            fn: set            
+          - name: project_id
+            fn: set

gen3.theanvil.io/manifest.json

@@ -7,27 +7,27 @@
     "autodeploy": "yes"
   },
   "versions": {
-    "arborist": "quay.io/cdis/arborist:2020.07",
+    "arborist": "quay.io/cdis/arborist:2020.08",
     "aws-es-proxy": "abutaha/aws-es-proxy:0.8",
-    "fence": "quay.io/cdis/fence:2020.07",
+    "fence": "quay.io/cdis/fence:4.23.1",
     "fluentd": "fluent/fluentd-kubernetes-daemonset:v1.2-debian-cloudwatch",
-    "guppy": "quay.io/cdis/guppy:2020.07",
-    "hatchery": "quay.io/cdis/hatchery:2020.07",
+    "guppy": "quay.io/cdis/guppy:2020.08",
+    "hatchery": "quay.io/cdis/hatchery:2020.08",
     "ambassador": "quay.io/datawire/ambassador:1.4.2",
-    "indexd": "quay.io/cdis/indexd:2020.07",
-    "manifestservice": "quay.io/cdis/manifestservice:2020.07",
-    "peregrine": "quay.io/cdis/peregrine:2020.07",
-    "pidgin": "quay.io/cdis/pidgin:2020.07",
-    "portal": "quay.io/cdis/data-portal:2020.07",
-    "revproxy": "quay.io/cdis/nginx:2020.07",
-    "sheepdog": "quay.io/cdis/sheepdog:2020.07",
-    "sower": "quay.io/cdis/sower:2020.07",
-    "spark": "quay.io/cdis/gen3-spark:2020.07",
-    "tube": "quay.io/cdis/tube:2020.07",
-    "ssjdispatcher": "quay.io/cdis/ssjdispatcher:2020.07",
-    "wts": "quay.io/cdis/workspace-token-service:2020.07",
-    "dashboard": "quay.io/cdis/gen3-statics:2020.07",
-    "metadata": "quay.io/cdis/metadata-service:2020.07"
+    "indexd": "quay.io/cdis/indexd:2020.08",
+    "manifestservice": "quay.io/cdis/manifestservice:2020.08",
+    "peregrine": "quay.io/cdis/peregrine:2020.08",
+    "pidgin": "quay.io/cdis/pidgin:2020.08",
+    "portal": "quay.io/cdis/data-portal:2020.08",
+    "revproxy": "quay.io/cdis/nginx:2020.08",
+    "sheepdog": "quay.io/cdis/sheepdog:2020.08",
+    "sower": "quay.io/cdis/sower:2020.08",
+    "spark": "quay.io/cdis/gen3-spark:2020.08",
+    "tube": "quay.io/cdis/tube:2020.08",
+    "ssjdispatcher": "quay.io/cdis/ssjdispatcher:2020.08",
+    "wts": "quay.io/cdis/workspace-token-service:2020.08",
+    "dashboard": "quay.io/cdis/gen3-statics:2020.08",
+    "metadata": "quay.io/cdis/metadata-service:2020.08"
   },
   "indexd": {
     "arborist": "true"
@@ -36,7 +36,7 @@
     "environment": "anvilprod",
     "hostname": "gen3.theanvil.io",
     "revproxy_arn": "arn:aws:acm:us-east-1:474789003679:certificate/9fd731e3-3366-4bd0-a3ef-0453dc07289a",
-    "dictionary_url": "https://s3.amazonaws.com/dictionary-artifacts/anvil/2.1.12/schema.json",
+    "dictionary_url": "https://s3.amazonaws.com/dictionary-artifacts/anvil/2.2.2/schema.json",
     "portal_app": "gitops",
     "kube_bucket": "kube-anvilprod-gen3",
     "logs_bucket": "logs-anvilprod-gen3",
@@ -57,7 +57,7 @@
       "action": "export",
       "container": {
         "name": "job-task",
-        "image": "quay.io/cdis/pelican-export:2020.07",
+        "image": "quay.io/cdis/pelican-export:2020.08",
         "pull_policy": "Always",
         "env": [
           {
@@ -122,7 +122,7 @@
   },
   "ssjdispatcher": {
     "job_images": {
-      "indexing": "quay.io/cdis/indexs3client:2020.07"
+      "indexing": "quay.io/cdis/indexs3client:2020.08"
     }
   },
   "arborist": {

gen3.theanvil.io/manifests/hatchery/hatchery.json

@@ -3,7 +3,7 @@
   "sub-dir": "/lw-workspace",
   "user-volume-size": "500Gi",
   "sidecar": {
-    "cpu-limit": "0.5",
+    "cpu-limit": "1.0",
     "memory-limit": "512Mi",
     "image": "quay.io/cdis/gen3fuse-sidecar:2020.07",
     "env": {
@@ -68,8 +68,8 @@
     },
     {
       "target-port": 8888,
-      "cpu-limit": "1.0",
-      "memory-limit": "6000Mi",
+      "cpu-limit": "4.0",
+      "memory-limit": "15512Mi",
       "name": "Jupyter Notebook Power Python",
       "image": "quay.io/occ_data/jupyternotebook:1.7.2",
       "env": {},