Deploy snpseq metadata service

env_vars/site_upps_env_all.yml

@@ -59,3 +59,7 @@ archive_upload_port: 10480
 # arteria archive verify
 archive_verify_port: 10490
 archive_verify_path_to_verify_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_dump/{{ deployment_environment }}"
+
+# snpseq-metadata-service
+snpseq_metadata_service_port: 10555
+snpseq_metadata_runfolder_path: "{{ static_runfolder_path }}"

install.yml

@@ -93,6 +93,7 @@
     - { role: arteria-checkqc-ws, tags: arteria-checkqc }
     - { role: arteria-delivery-ws, tags: arteria-delivery }
     - { role: arteria-sequencing-report-ws, tags: arteria-sequencing-report }
+    - { role: snpseq-metadata-service, tags: snpseq-metadata-service }
     - { role: standalone_scripts, tags: standalone_scripts }
     - { role: delivery, tags: delivery }
     - { role: misc-tools, tags: misc-tools }

roles/snpseq-metadata-service/defaults/main.yml

@@ -0,0 +1,16 @@
+snpseq_metadata_env_name: snpseq_metadata
+snpseq_metadata_env_path: "{{ anaconda_path }}/envs/{{ snpseq_metadata_env_name }}"
+
+snpseq_metadata_repo: https://github.com/Molmed/snpseq_metadata.git
+snpseq_metadata_version: 2763cee
+xml_schema_source_url: ftp://ftp.ebi.ac.uk/pub/databases/ena/doc/xsd/sra_1_5
+
+snpseq_metadata_src_path: "{{ sw_path }}/metadata/snpseq_metadata"
+
+snpseq_metadata_service_repo: https://github.com/Molmed/snpseq-metadata-service.git
+snpseq_metadata_service_version: 39752ef
+
+snpseq_metadata_service_log_dir: "{{ static_ngi_log_path }}/metadata/snpseq_metadata_service/{{ deployment_environment }}"
+snpseq_metadata_service_src_path: "{{ sw_path }}/metadata/snpseq_metadata_service"
+snpseq_metadata_service_config_root: "{{ ngi_pipeline_conf }}/metadata/snpseq_metadata_service"
+

roles/snpseq-metadata-service/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - { role: anaconda, tags: anaconda }
+  - { role: setup_base_config, tags: setup_base_config }

roles/snpseq-metadata-service/tasks/install_config.yml

@@ -0,0 +1,32 @@
+---
+
+- name: ensure snpseq-metadata-service conf dir exists
+  file:
+    state: directory
+    path: "{{ snpseq_metadata_service_config_root }}"
+
+- name: deploy snpseq-metadata-service app config
+  template:
+    src: snpseq_metadata_service_app.config.j2
+    dest: "{{ snpseq_metadata_service_config_root }}/app.yaml"
+
+- name: deploy snpseq-metadata-service logger config
+  template:
+    src: snpseq_metadata_service_logger.config.j2
+    dest: "{{ snpseq_metadata_service_config_root }}/logger.yaml"
+
+- name: modify uppsala's supervisord conf to set environment and program commands
+  ini_file:
+    dest: "{{ supervisord_conf }}"
+    section: program:snpseq-metadata-service-{{ deployment_environment }}
+    option: "{{ item.opt }}"
+    value: "{{ item.val }}"
+    backup: no
+  loop:
+    - opt: command
+      val: "{{ snpseq_metadata_env_path}}/bin/metadata-service -c {{ snpseq_metadata_service_config_root }}"
+    - opt: autorestart
+      val: true
+    - opt: directory
+      val: "{{ snpseq_metadata_service_log_dir }}"
+

roles/snpseq-metadata-service/tasks/install_package.yml

@@ -0,0 +1,26 @@
+
+- name: create virtual python env, shared with snpseq-metadata-service
+  shell: "conda create --name {{ snpseq_metadata_env_name }} python={{ python3_version }}"
+  args:
+    creates: "{{ snpseq_metadata_env_path }}"
+
+- name: get snpseq_metadata from git
+  git:
+    repo: "{{ snpseq_metadata_repo }}"
+    dest: "{{ snpseq_metadata_src_path }}"
+    version: "{{ snpseq_metadata_version }}"
+
+- name: install snpseq_metadata
+  pip:
+    name: .
+    chdir: "{{ snpseq_metadata_src_path }}"
+    executable: "{{ snpseq_metadata_env_path }}/bin/pip"
+    state: present
+    extra_args: "-U -e"
+
+- name: download XML schema
+  command: "{{ snpseq_metadata_env_path }}/bin/generate_python_models.sh {{ snpseq_metadata_env_path }}/bin/xsdata {{ xml_schema_source_url }}"
+  environment:
+    PATH: "{{ snpseq_metadata_env_path }}/bin"
+  args:
+    chdir: "{{ snpseq_metadata_src_path }}"

roles/snpseq-metadata-service/tasks/install_service.yml

@@ -0,0 +1,19 @@
+
+- name: create virtual python env, shared with snpseq_metadata
+  shell: "conda create --name {{ snpseq_metadata_env_name }} python={{ python3_version }}"
+  args:
+    creates: "{{ snpseq_metadata_env_path }}"
+
+- name: get snpseq-metadata-service from git
+  git:
+    repo: "{{ snpseq_metadata_service_repo }}"
+    dest: "{{ snpseq_metadata_service_src_path }}"
+    version: "{{ snpseq_metadata_service_version }}"
+
+- name: install snpseq-metadata-service
+  pip:
+    name: .
+    chdir: "{{ snpseq_metadata_service_src_path }}"
+    executable: "{{ snpseq_metadata_env_path }}/bin/pip"
+    state: present
+    extra_args: "-U"

roles/snpseq-metadata-service/tasks/main.yml

@@ -0,0 +1,21 @@
+
+- block:
+  - include: install_package.yml
+  - include: install_service.yml
+  - include: install_config.yml
+
+  - name: add static folders to be created
+    set_fact:
+      static_folders: "{{ static_folders + [snpseq_metadata_runfolder_path, snpseq_metadata_service_log_dir] }}"
+
+  - name: Store tool versions in deployment
+    lineinfile:
+      dest: "{{ deployed_tool_versions }}"
+      line: "{{ item.tool }}: {{ item.ver }}"
+    loop:
+      - tool: snpseq-metadata-service
+        ver: "{{ snpseq_metadata_service_version }}"
+      - tool: snpseq_metadata
+        ver: "{{ snpseq_metadata_version }}"
+
+  when: site in ["upps"]

roles/snpseq-metadata-service/templates/snpseq_metadata_service_app.config.j2

@@ -0,0 +1,18 @@
+
+# the service will listen on this port
+port: {{ snpseq_metadata_service_port }}
+
+# the base url preceding the service endpoint
+base_url: /api/1.0
+
+# the location on the server under which runfolders are stored
+# {runfolder} and {host} are substituted at each request to snpseq-metadata-server
+datadir: "{{ snpseq_metadata_runfolder_path }}/{runfolder}"
+
+# the url where the snpseq-data service can be accessed
+# on miarka, it won't be used so let's just point back to kong
+snpseq_data_url: http://localhost:4444
+
+# the path to the snpseq_metadata python package main script
+# will be deployed in the same env as the service
+snpseq_metadata_executable: snpseq_metadata

roles/snpseq-metadata-service/templates/snpseq_metadata_service_logger.config.j2

@@ -0,0 +1,28 @@
+---
+version: 1
+
+disable_existing_loggers: False
+
+formatters:
+    simple:
+        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+handlers:
+    console:
+        class: logging.StreamHandler
+        level: DEBUG
+        formatter: simple
+        stream: ext://sys.stdout
+
+    file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: DEBUG  # By default set to WARNING, but can be changed on runtime
+        formatter: simple
+        filename: "{{ snpseq_metadata_service_log_dir }}/snpseq_metadata_service.log"
+        maxBytes: 10485760  # 10MB
+        backupCount: 20
+        encoding: utf8
+
+root:
+    level: DEBUG
+    handlers: [console, file_handler]

roles/tarzan/templates/kong.yml.j2

@@ -50,6 +50,14 @@ services:
       - /checkqc
       strip_path: true
 
+  - name: snpseq_metadata
+    host: localhost
+    port: {{ snpseq_metadata_port }}
+    routes:
+    - paths:
+      - /metadata
+      strip_path: true
+
 plugins:
   - name: key-auth
     config: