feat!: switch to Phenopackets v2 #1394

.env-sample

@@ -5,6 +5,7 @@ export SERVICE_SECRET_KEY=some_secret_key
 export POSTGRES_DATABASE=
 export POSTGRES_USER=
 export POSTGRES_PASSWORD=
+
 # Should only need to use a host or socket dir
 export POSTGRES_HOST=
 export POSTGRES_SOCKET_DIR=

.github/workflows/build.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     branches:
       - develop
+      - 'features/**'
   push:
     branches:
       - develop
@@ -24,7 +25,7 @@ jobs:
           submodules: "recursive"
 
       - name: Run Bento build action
-        uses: bento-platform/bento_build_action@v0.11.0
+        uses: bento-platform/bento_build_action@v0.11.1
         with:
           registry: ghcr.io
           registry-username: ${{ github.actor }}

.github/workflows/lint.yml

@@ -9,11 +9,11 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         name: Set up Python
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install flake8
         run: python -m pip install flake8
       - name: Run linter

.github/workflows/test.yml

@@ -7,6 +7,7 @@ on:
       - master
   pull_request:
     branches:
+      - 'features/**'
       - develop
       - master
 
@@ -16,10 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.10", "3.11"]
     services:
       postgres:
-        image: postgres:13
+        image: postgres:15
         env:
           POSTGRES_DB: postgres
           POSTGRES_PASSWORD: postgres
@@ -29,7 +30,7 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Set up Python ${{ matrix.python-version }}
@@ -43,6 +44,6 @@ jobs:
     - name: Test
       run: |
         export POSTGRES_USER="postgres" && export POSTGRES_PASSWORD="postgres" && export POSTGRES_PORT=5432
-        coverage run ./manage.py test
+        poetry run coverage run ./manage.py test
     - name: Upload Coverage to Codecov
       uses: codecov/codecov-action@v3

.gitignore

@@ -5,6 +5,9 @@ __pycache__/
 # sqlite
 /*.sqlite3
 
+#PostgreSQL
+postgres/
+
 # migrations
 patients/migrations/*
 

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "chord_metadata_service/dats"]
 	path = chord_metadata_service/dats
 	url = https://github.com/datatagsuite/schema
+[submodule "chord_metadata_service/vrs"]
+	path = chord_metadata_service/vrs
+	url = https://github.com/ga4gh/vrs

README.md

@@ -11,23 +11,30 @@ A Phenopackets-based clinical and phenotypic metadata service for the Bento plat
 
 ## Table of Contents
 
-  * [License](#license)
-  * [Funding](#funding)
-  * [Architecture](#architecture)
-  * [REST API Highlights](#rest-api-highlights)
-  * [Install](#install)
-  * [Authentication](#authentication)
-     * [Note on Permissions](#note-on-permissions)
-     * [Authorization inside CanDIG](#authorization-inside-candig)
-  * [Developing](#developing)
-     * [Branching](#branching)
-     * [Tests](#tests)
-     * [Terminal Commands](#terminal-commands)
-        * [Project/Dataset/Table/Ingestion Commands](#projectdatasettableingestion-commands)
-        * [Patient Commands](#patient-commands)
-        * [Phenopacket Commands](#phenopacket-commands)
-     * [Accessing the Django Shell from inside a Bento Container](#accessing-the-django-shell-from-inside-a-bento-container)
-  * [Configuring Public overview and public search fields](#configuring-public-overview-and-public-search-fields)
+- [Katsu Metadata Service](#katsu-metadata-service)
+  - [Table of Contents](#table-of-contents)
+  - [License](#license)
+  - [Funding](#funding)
+  - [Architecture](#architecture)
+  - [REST API highlights](#rest-api-highlights)
+  - [Install](#install)
+    - [Install via Docker](#install-via-docker)
+  - [Environment Variables](#environment-variables)
+  - [Standalone PostGres db and AdMiner](#standalone-postgres-db-and-adminer)
+  - [Authentication](#authentication)
+    - [Note on Permissions](#note-on-permissions)
+    - [Authorization inside CanDIG](#authorization-inside-candig)
+  - [Developing](#developing)
+    - [Branching](#branching)
+    - [Tests](#tests)
+    - [Terminal Commands](#terminal-commands)
+      - [Project/Dataset/Table/Ingestion Commands](#projectdatasettableingestion-commands)
+      - [Patient Commands](#patient-commands)
+      - [Phenopacket Commands](#phenopacket-commands)
+    - [Accessing the Django Shell from inside a Bento Container](#accessing-the-django-shell-from-inside-a-bento-container)
+      - [When running Katsu with `bentoV2`:](#when-running-katsu-with-bentov2)
+      - [When running Katsu with `chord_singularity` (DEPRECATED)](#when-running-katsu-with-chord_singularity-deprecated)
+  - [Configuring public overview and public search fields](#configuring-public-overview-and-public-search-fields)
 
 ## License
 
@@ -92,10 +99,9 @@ git submodule update --init
 
 The service uses PostgreSQL database for data storage.
 
-* Create and activate virtual environment
-* Install Poetry (for dependency management) in the virtual environment: `pip install poetry`
-* Install dependencies with `poetry install`
-* To configure the application (such as the DB credentials) we are using python-dotenv:
+* Install Poetry (for dependency management): `pip install poetry`
+* Install dependencies with `poetry install`, which manages its own virtual environment
+* To configure the application (such as the DB credentials) we are using `python-dotenv`:
     - Take a look at the .env-sample file at the root of the project
     - You can export these in your virtualenv or simply `cp .env-sample .env`
     - `python-dotenv` can handle either (a local .env will override environment variables though)
@@ -104,18 +110,20 @@ The service uses PostgreSQL database for data storage.
 * Run:
 
 ```
-python manage.py makemigrations
-python manage.py migrate
-python manage.py runserver
+poetry run python manage.py makemigrations
+poetry run python manage.py migrate
+poetry run python manage.py runserver
 ```
 
 * Development server runs at `localhost:8000`
 
 
 ### Install via Docker
 
-Optionally, you may also install standalone Katsu with the Dockerfile provided. If you develop or
-deploy Katsu as part of the Bento platform, you should use Bento's Docker image instead.
+Optionally, you may also install standalone Katsu with the Dockerfile provided. If you develop or deploy Katsu as 
+part of the Bento platform, you should use Bento's Docker 
+[image](https://github.com/bento-platform/katsu/pkgs/container/katsu) instead.
+
 
 
 ## Environment Variables
@@ -162,6 +170,29 @@ CANDIG_OPA_SITE_ADMIN_KEY=
 INSIDE_CANDIG=
 ```
 
+## Standalone Postgres db and Adminer
+
+For local development, you can quickly deploy a local database server (Postgres) and management tool (Adminer) with 
+`docker compose`. Make sure your Postgres env variables are set in `.env` before running:
+
+```
+# Start docker compose containers
+docker compose -f docker-compose.dev.yaml up -d
+
+# Stop and remove docker-compose containers
+docker compose -f docker-compose.dev.yaml down
+```
+
+You can now use the katsu-db container (`localhost:5432`) as your database for standalone katsu development and 
+explore the database tables with adminer (`localhost:8080`).
+Login to adminer by specifying the following on the login page:
+
+- **System:** `PostgreSQL`
+- **Server:** `katsu-db` (host and port are resolved by Docker with the container name)
+- **Username:** `POSTGRES_USER`
+- **Password:** `POSTGRES_PASSWORD`
+- **Database:** `POSTGRES_DATABASE`
+
 ## Authentication
 
 Default authentication can be set globally in `settings.py`

bento.Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.03.22
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.10.20
 
 SHELL ["/bin/bash", "-c"]
 
@@ -7,17 +7,17 @@ SHELL ["/bin/bash", "-c"]
 RUN apt-get update -y && \
     apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install --no-cache-dir "uvicorn[standard]==0.20.0"
+    pip install --no-cache-dir "uvicorn[standard]==0.23.2"
 
 # Backwards-compatible with old BentoV2 container layout
 WORKDIR /app
 
 COPY pyproject.toml .
 COPY poetry.lock .
-COPY poetry.toml .
 
 # Install production dependencies
-RUN poetry install --no-root --without dev
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root --without dev
 
 # Copy all application code
 COPY . .

bento.dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.03.22
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.10.20
 
 LABEL org.opencontainers.image.description="Local development image for Katsu."
 LABEL devcontainer.metadata='[{ \
@@ -18,19 +18,19 @@ SHELL ["/bin/bash", "-c"]
 RUN apt-get update -y && \
     apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install --no-cache-dir "uvicorn[standard]==0.20.0"
+    pip install --no-cache-dir "uvicorn[standard]==0.23.2"
 
 # Backwards-compatible with old BentoV2 container layout
 WORKDIR /app
 
 COPY pyproject.toml .
 COPY poetry.lock .
-COPY poetry.toml .
 
 # Install production + development dependencies
 # Without --no-root, we get errors related to the code not being copied in yet.
 # But we don't want the code here, otherwise Docker cache doesn't work well.
-RUN poetry install --no-root
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root
 
 # Create temporary directory for downloading files etc.
 RUN mkdir -p /app/tmp
@@ -39,5 +39,8 @@ RUN mkdir -p /app/tmp
 COPY entrypoint.bash .
 COPY run.dev.bash .
 
+# Tell the service that we're running a local development container
+ENV BENTO_CONTAINER_LOCAL=True
+
 ENTRYPOINT [ "bash", "./entrypoint.bash" ]
 CMD [ "bash", "./run.dev.bash" ]

chord_metadata_service/chord/data_types.py

@@ -2,21 +2,16 @@
 
 from chord_metadata_service.experiments.search_schemas import EXPERIMENT_SEARCH_SCHEMA
 from chord_metadata_service.phenopackets.search_schemas import PHENOPACKET_SEARCH_SCHEMA
-from chord_metadata_service.experiments.schemas import EXPERIMENT_RESULT_SCHEMA
 
 __all__ = [
     "DATA_TYPE_EXPERIMENT",
-    "DATA_TYPE_EXPERIMENT_RESULT",
     "DATA_TYPE_PHENOPACKET",
-    "DATA_TYPE_READSET",
     "DATA_TYPES",
 ]
 
 
 DATA_TYPE_EXPERIMENT = "experiment"
-DATA_TYPE_EXPERIMENT_RESULT = "experiment_result"
 DATA_TYPE_PHENOPACKET = "phenopacket"
-DATA_TYPE_READSET = "readset"
 
 DATA_TYPES = {
     DATA_TYPE_EXPERIMENT: {
@@ -35,22 +30,4 @@
             "type": "object",  # TODO
         },
     },
-    DATA_TYPE_READSET: {
-        "label": "Readsets",
-        "queryable": False,
-        "schema": {
-            "file_format": EXPERIMENT_RESULT_SCHEMA["properties"]["file_format"]
-        },
-        "metadata_schema": {
-            "type": "object"
-        }
-    },
-    DATA_TYPE_EXPERIMENT_RESULT: {
-        "label": "Experiment Results",
-        "queryable": False,
-        "schema": EXPERIMENT_RESULT_SCHEMA,
-        "metadata_schema": {
-            "type": "object"
-        }
-    }
 }

chord_metadata_service/chord/ingest/constants.py

@@ -0,0 +1,20 @@
+from typing import Callable
+
+from chord_metadata_service.chord.data_types import DATA_TYPE_EXPERIMENT, DATA_TYPE_PHENOPACKET
+from chord_metadata_service.chord.ingest.experiments import ingest_experiments_workflow, validate_experiment
+from chord_metadata_service.chord.ingest.phenopackets import ingest_phenopacket_workflow, validate_phenopacket
+
+__all__ = [
+    "DATA_TYPE_TO_VALIDATOR_FN"
+]
+
+
+DATA_TYPE_TO_VALIDATOR_FN: dict[str, Callable] = {
+    DATA_TYPE_EXPERIMENT: validate_experiment,
+    DATA_TYPE_PHENOPACKET: validate_phenopacket,
+}
+
+DATA_TYPE_TO_INGESTION_FN: dict[str, Callable] = {
+    DATA_TYPE_EXPERIMENT: ingest_experiments_workflow,
+    DATA_TYPE_PHENOPACKET: ingest_phenopacket_workflow,
+}