Don't run docker containers as root

.env.sample

@@ -103,3 +103,7 @@ HASHER_API_AZ_CLIENT_ID=
 HASHER_API_AZ_CLIENT_PASSWORD=
 HASHER_API_AZ_TENANT_ID=
 HASHER_API_AZ_KEY_VAULT_NAME=
+
+# User and group IDs for pixl:pixl. Only needed at build time.
+PIXL_USER_UID=7001
+PIXL_USER_GID=7001

.github/workflows/main.yml

@@ -108,15 +108,6 @@ jobs:
             - name: Create .secrets.env
               run: touch test/.secrets.env
 
-            - name: Build test services
-              working-directory: test
-              run: |
-                  docker compose build
-
-            - name: Build services
-              run: |
-                  docker compose build
-
             - name: Run tests
               working-directory: test
               env:
@@ -129,7 +120,24 @@ jobs:
                   HASHER_API_AZ_TENANT_ID: ${{ secrets.EXPORT_AZ_TENANT_ID }}
                   HASHER_API_AZ_KEY_VAULT_NAME: ${{ secrets.EXPORT_AZ_KEY_VAULT_NAME }}
               run: |
-                  ./run-system-test.sh coverage
+                  # We need PIXL_USER_UID and PIXL_USER_GID from the test env file.
+                  source .env
+                  # Set up as per docker_perms.md
+                  # Most importantly, the pixl user is *not* in the docker group, and
+                  # the default "runner" user is added to the pixl group.
+                  sudo --non-interactive groupadd --gid $PIXL_USER_GID pixl
+                  sudo --non-interactive useradd --uid $PIXL_USER_UID --gid pixl pixl
+                  sudo --non-interactive usermod --append --groups pixl runner
+                  host_export_root_dir=../projects/exports
+                  sudo --non-interactive chown -R pixl:pixl "$host_export_root_dir"
+                  sudo --non-interactive chmod -R u=rwx,g=rwxs,o= "$host_export_root_dir"
+                  sudo --non-interactive setfacl -R -m d:g::rwX  "$host_export_root_dir"
+                  # The top-level commands are still run as the GHA "runner" user, which
+                  # is analogous to a human user on the GAE bringing up a PIXL instance.
+                  # The sudo command is needed to make it run in a new shell
+                  # so that the new group (see usermod above) is picked up.
+                  # PATH needs to be explicitly mentioned to be passed through.
+                  sudo --non-interactive --preserve-env --preserve-env=PATH -u runner ./run-system-test.sh coverage
                   echo FINISHED SYSTEM TEST SCRIPT
 
             - name: Dump queue docker logs for debugging
@@ -178,6 +186,12 @@ jobs:
                   docker logs -t system-test-dicomweb-server-1 2>&1
 
 
+            - name: View filesystem for debugging
+              if: ${{ failure() }}
+              run: |
+                  # just in case the error is caused by permissions, make sure all files are seen
+                  sudo --non-interactive  ls -laR
+
             - name: Upload coverage reports to Codecov
               uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
               with:

README.md

@@ -282,10 +282,14 @@ These files will subsequently be uploaded to the `parquet` destination specified
 [project config](#3-configure-a-new-project).
 
 ```
-EXPORT_ROOT/PROJECT_SLUG/all_extracts/EXTRACT_DATETIME/radiology/radiology.parquet
+EXPORT_ROOT/PROJECT_SLUG/all_extracts/EXTRACT_DATETIME/radiology/IMAGE_LINKER.parquet
 ....................................................../omop/public/*.parquet
 ```
 
+This directory is currently located in `projects/exports` relative to the source code root.
+
+[Further discussion about docker permissions and the exports directory is found here](docs/setup/docker_perms.md)
+
 ### Destination
 
 #### FTP server

docker-compose.yml

@@ -25,8 +25,12 @@ x-proxy-common: &proxy-common
     NO_PROXY: *no-proxy
     no_proxy: *no-proxy
 
+x-pixl-uid-gid: &pixl-uid-gid
+    PIXL_USER_UID: ${PIXL_USER_UID}
+    PIXL_USER_GID: ${PIXL_USER_GID}
+
 x-build-args-common: &build-args-common
-    <<: [*proxy-common]
+    <<: [*proxy-common, *pixl-uid-gid]
 
 x-pixl-common-env: &pixl-common-env
     DEBUG: ${DEBUG}
@@ -74,8 +78,10 @@ services:
     hasher-api:
         build:
             context: .
-            dockerfile: ./docker/hasher-api/Dockerfile
+            dockerfile: ./docker/pixl-python/Dockerfile
+            target: hasher_api
             args:
+                PIXL_PACKAGE_DIR: hasher
                 <<: *build-args-common
         environment:
             <<: [*proxy-common, *pixl-common-env]
@@ -93,7 +99,6 @@ services:
         networks:
             - pixl-net
         healthcheck:
-            test: ["CMD", "curl", "-f", "http://hasher-api:8000/heart-beat"]
             interval: 10s
             timeout: 30s
             retries: 5
@@ -102,7 +107,8 @@ services:
     orthanc-anon:
         build:
             context: .
-            dockerfile: ./docker/orthanc-anon/Dockerfile
+            dockerfile: ./docker/orthanc/Dockerfile
+            target: pixl_orthanc_anon
             args:
                 <<: *build-args-common
                 ORTHANC_CONCURRENT_JOBS: ${ORTHANC_CONCURRENT_JOBS}
@@ -167,7 +173,8 @@ services:
     orthanc-raw:
         build:
             context: .
-            dockerfile: ./docker/orthanc-raw/Dockerfile
+            dockerfile: ./docker/orthanc/Dockerfile
+            target: pixl_orthanc_raw
             args:
                 <<: *build-args-common
                 ORTHANC_RAW_MAXIMUM_STORAGE_SIZE: ${ORTHANC_RAW_MAXIMUM_STORAGE_SIZE}
@@ -245,8 +252,10 @@ services:
     export-api:
         build:
             context: .
-            dockerfile: ./docker/export-api/Dockerfile
+            dockerfile: ./docker/pixl-python/Dockerfile
+            target: export_api
             args:
+                PIXL_PACKAGE_DIR: pixl_export
                 <<: *build-args-common
         environment:
             <<:
@@ -285,29 +294,30 @@ services:
         extra_hosts:
             - "host.docker.internal:host-gateway"
         volumes:
-            - ${PWD}/projects/exports:/run/projects/exports
+            - ${PWD}/projects/exports:/run/projects/exports:ro
             - ${PWD}/projects/configs:/${PROJECT_CONFIGS_DIR:-/projects/configs}:ro
 
     imaging-api:
         build:
             context: .
-            dockerfile: ./docker/imaging-api/Dockerfile
+            dockerfile: ./docker/pixl-python/Dockerfile
+            target: imaging_api
             args:
+                PIXL_PACKAGE_DIR: pixl_imaging
                 <<: *build-args-common
         depends_on:
             queue:
                 condition: service_healthy
             orthanc-raw:
                 condition: service_healthy
         healthcheck:
-            test: curl -f http://0.0.0.0:8000/heart-beat
             interval: 10s
             timeout: 30s
             retries: 5
         networks:
             - pixl-net
         environment:
-            <<: [*pixl-rabbit-mq, *proxy-common, *pixl-common-env]
+            <<: [*pixl-rabbit-mq, *proxy-common, *pixl-common-env, *pixl-db]
             ORTHANC_RAW_URL: ${ORTHANC_RAW_URL}
             ORTHANC_RAW_USERNAME: ${ORTHANC_RAW_USERNAME}
             ORTHANC_RAW_PASSWORD: ${ORTHANC_RAW_PASSWORD}
@@ -316,11 +326,6 @@ services:
             PRIMARY_DICOM_SOURCE_MODALITY: ${PRIMARY_DICOM_SOURCE_MODALITY}
             SECONDARY_DICOM_SOURCE_MODALITY: ${SECONDARY_DICOM_SOURCE_MODALITY}
             SKIP_ALEMBIC: ${SKIP_ALEMBIC}
-            PIXL_DB_HOST: ${PIXL_DB_HOST}
-            PIXL_DB_PORT: ${PIXL_DB_PORT}
-            PIXL_DB_NAME: ${PIXL_DB_NAME}
-            PIXL_DB_USER: ${PIXL_DB_USER}
-            PIXL_DB_PASSWORD: ${PIXL_DB_PASSWORD}
             PIXL_DICOM_TRANSFER_TIMEOUT: ${PIXL_DICOM_TRANSFER_TIMEOUT}
             PIXL_QUERY_TIMEOUT: ${PIXL_QUERY_TIMEOUT}
             PIXL_MAX_MESSAGES_IN_FLIGHT: ${PIXL_MAX_MESSAGES_IN_FLIGHT}