diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4399b7459..06e265af5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,6 +20,11 @@ on: pull_request: workflow_dispatch: +# Only run actions on the most recent push to a branch +concurrency: + group: "${{ github.workflow }}-${{ github.head_ref }}" + cancel-in-progress: true + jobs: lint: @@ -154,8 +159,6 @@ jobs: - name: Run tests working-directory: pixl_ehr/tests - env: - INFORMDB_PAT: ${{ secrets.INFORMDB_PAT }} run: | ./run-tests.sh @@ -179,3 +182,20 @@ jobs: working-directory: pixl_pacs/tests run: | ./run-tests.sh + + system-test: + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v3 + + - name: Init Python + uses: actions/setup-python@v4 + with: + python-version: 3.10.6 + cache: 'pip' + + - name: Run tests + working-directory: test + run: | + ./run-system-test.sh diff --git a/pixl_core/src/core/patient_queue/subscriber.py b/pixl_core/src/core/patient_queue/subscriber.py index 03805ef8c..21c41453e 100644 --- a/pixl_core/src/core/patient_queue/subscriber.py +++ b/pixl_core/src/core/patient_queue/subscriber.py @@ -69,13 +69,14 @@ async def run(self, callback: Callable[[Message], Awaitable[None]]) -> None: await message.reject(requeue=True) continue + pixl_message = deserialise(message.body) try: await asyncio.sleep(0.01) # Avoid very fast callbacks - await callback(deserialise(message.body)) + await callback(pixl_message) except Exception: logger.exception( "Failed to process %s" "Not re-queuing message", - message.body.decode(), + pixl_message, ) finally: await message.ack() diff --git a/pixl_ehr/tests/docker-compose.yml b/pixl_ehr/tests/docker-compose.yml index a89feec36..47d29400c 100644 --- a/pixl_ehr/tests/docker-compose.yml +++ b/pixl_ehr/tests/docker-compose.yml @@ -97,7 +97,6 @@ services: dockerfile: ../../test/Dockerfile.fake_emap_star args: N_TABLE_ROWS: 2 - INFORMDB_PAT: ${INFORMDB_PAT} healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] diff --git a/pixl_ehr/tests/run-processing-tests.sh b/pixl_ehr/tests/run-processing-tests.sh index 9afbef241..7b6f93184 100755 --- a/pixl_ehr/tests/run-processing-tests.sh +++ b/pixl_ehr/tests/run-processing-tests.sh @@ -32,10 +32,7 @@ THIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${THIS_DIR%/*}" cd "$PACKAGE_DIR"/tests || exit -if [ -z "${INFORMDB_PAT+x}" ]; then - echo "INFORMDB_PAT must be set as an environment variable" && exit 1 -fi - +docker compose down --volumes docker compose up -d --build docker exec pixl-test-ehr-api /bin/bash -c "pytest -m processing" -docker compose down +docker compose down --volumes diff --git a/test/.env.test.sample b/test/.env.test similarity index 96% rename from test/.env.test.sample rename to test/.env.test index 43413d509..546ecc77e 100644 --- a/test/.env.test.sample +++ b/test/.env.test @@ -3,8 +3,6 @@ DEBUG=True PIXL_DICOM_TRANSFER_TIMEOUT=120 PIXL_QUERY_TIMEOUT=120 -INFORMDB_PAT= - # PIXL PostgreSQL instance PIXL_DB_HOST=postgres PIXL_DB_PORT=5432 @@ -37,7 +35,7 @@ ORTHANC_RAW_USERNAME=orthanc_raw_username ORTHANC_RAW_PASSWORD=orthanc_raw_password ORTHANC_RAW_AE_TITLE=ORTHANCRAW ORTHANC_AUTOROUTE_RAW_TO_ANON=true -ORTHANC_RAW_MAXIMUM_STORAGE_SIZE=1 +ORTHANC_RAW_MAXIMUM_STORAGE_SIZE=100 # PIXL Orthanc anon instance ORTHANC_ANON_USERNAME=orthanc_anon_username diff --git a/test/.gitignore b/test/.gitignore deleted file mode 100644 index ed9875073..000000000 --- a/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.env.test diff --git a/test/Dockerfile.fake_emap_star b/test/Dockerfile.fake_emap_star index 65987f46f..1a0275b28 100644 --- a/test/Dockerfile.fake_emap_star +++ b/test/Dockerfile.fake_emap_star @@ -14,7 +14,6 @@ FROM postgres:15.0-bullseye # User definable arguments -ARG INFORMDB_PAT ARG POSTGRES_USER=postgres ARG POSTGRES_PASSWORD=postgres ARG DATABASE_NAME=emap @@ -22,7 +21,7 @@ ARG N_TABLE_ROWS=2 ARG INSERT_RATE=0.3 ARG UPDATE_RATE=0.0 ARG DELETE_RATE=0.0 -ARG INFORMDB_BRANCH_NAME=develop +ARG EMAP_BRANCH_NAME=main ARG STAR_SCHEMA_NAME=star ARG FAKER_SEED=0 ARG TIMEZONE="Europe/London" @@ -57,14 +56,13 @@ RUN pip install --no-cache-dir . && \ ENV POSTGRES_USER ${POSTGRES_USER} ENV POSTGRES_PASSWORD ${POSTGRES_PASSWORD} ENV DATABASE_NAME ${DATABASE_NAME} -ENV INFORMDB_BRANCH_NAME ${INFORMDB_BRANCH_NAME} +ENV EMAP_BRANCH_NAME ${EMAP_BRANCH_NAME} ENV STAR_SCHEMA_NAME ${STAR_SCHEMA_NAME} ENV FAKER_SEED ${FAKER_SEED} ENV TIMEZONE ${TIMEZONE} ENV INSERT_RATE ${INSERT_RATE} ENV UPDATE_RATE ${UPDATE_RATE} ENV DELETE_RATE ${DELETE_RATE} -ENV INFORMDB_PAT ${INFORMDB_PAT} ENV LANG=en_GB.UTF-8 ENV LC_ALL=en_GB.UTF-8 diff --git a/test/README.md b/test/README.md index d551a4f18..623a1a261 100644 --- a/test/README.md +++ b/test/README.md @@ -11,16 +11,7 @@ queue and the consumers started. **Then** a row in the "anon" EMAP data instance of the PIXL postgres instance exists and the DICOM study exists in the "anon" PIXL Orthanc instance. -Spinning up the Docker containers requires the `$INFORMDB_PAT` environment variable to be set to a -valid [GitHub Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) -with read access to the [UCLH-Foundry/Inform-DB](https://github.com/UCLH-Foundry/Inform-DB) repo. - -1. Create a [fine-grained Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) - with **read** access to the *UCLH-Foundry/Inform-DB* repo -2. Create `.env.test` with `cp .env.test.sample .env.test` -3. Add your `githb_pat***` token to `.env.test` as `INFORMDB_PAT=***` - -Then, run the system test with: +You can run the system test with: ```bash ./run-system-test.sh diff --git a/test/data/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet b/test/data/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet deleted file mode 100644 index 97a953bd4..000000000 Binary files a/test/data/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet and /dev/null differ diff --git a/test/data/resources/omop/public/PROCEDURE_OCCURRENCE.parquet b/test/data/resources/omop/public/PROCEDURE_OCCURRENCE.parquet deleted file mode 100644 index 01451206e..000000000 Binary files a/test/data/resources/omop/public/PROCEDURE_OCCURRENCE.parquet and /dev/null differ diff --git a/test/data/test.dcm b/test/data/test.dcm deleted file mode 100644 index 2784caa2f..000000000 Binary files a/test/data/test.dcm and /dev/null differ diff --git a/test/docker-compose.yml b/test/docker-compose.yml index b1660b810..e0e6bcd8a 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -36,17 +36,21 @@ services: - "8043:8042" volumes: - ${PWD}/vna_config/:/run/secrets:ro + healthcheck: + test: [ "CMD", "curl", "-f", "-u" , "orthanc:orthanc", "http://vna-qr:8042/patients" ] + interval: 10s + timeout: 30s + retries: 5 networks: pixl-net: star: - container_name: test-fake-star-db + container_name: system-test-fake-star-db build: context: .. dockerfile: test/Dockerfile.fake_emap_star args: N_TABLE_ROWS: 2 - INFORMDB_PAT: ${INFORMDB_PAT} environment: POSTGRES_USER: ${EMAP_UDS_USER} POSTGRES_PASSWORD: ${EMAP_UDS_PASSWORD} diff --git a/test/resources/Dicom1.dcm b/test/resources/Dicom1.dcm new file mode 100644 index 000000000..c6f3d4c19 Binary files /dev/null and b/test/resources/Dicom1.dcm differ diff --git a/test/resources/Dicom2.dcm b/test/resources/Dicom2.dcm new file mode 100644 index 000000000..89fc0815b Binary files /dev/null and b/test/resources/Dicom2.dcm differ diff --git a/test/data/resources/omop/extract_summary.json b/test/resources/omop/extract_summary.json similarity index 100% rename from test/data/resources/omop/extract_summary.json rename to test/resources/omop/extract_summary.json diff --git a/test/data/resources/omop/private/PERSON_LINKS.parquet b/test/resources/omop/private/PERSON_LINKS.parquet similarity index 100% rename from test/data/resources/omop/private/PERSON_LINKS.parquet rename to test/resources/omop/private/PERSON_LINKS.parquet diff --git a/test/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet b/test/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet new file mode 100644 index 000000000..efef741a4 Binary files /dev/null and b/test/resources/omop/private/PROCEDURE_OCCURRENCE_LINKS.parquet differ diff --git a/test/resources/omop/public/PROCEDURE_OCCURRENCE.parquet b/test/resources/omop/public/PROCEDURE_OCCURRENCE.parquet new file mode 100644 index 000000000..0b526f617 Binary files /dev/null and b/test/resources/omop/public/PROCEDURE_OCCURRENCE.parquet differ diff --git a/test/run-system-test.sh b/test/run-system-test.sh index dc70cd51d..c904a7517 100755 --- a/test/run-system-test.sh +++ b/test/run-system-test.sh @@ -17,16 +17,19 @@ BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${BIN_DIR%/*}" cd "${PACKAGE_DIR}/test" -# Note: this doesn't work as a single command -docker compose --env-file .env.test -p test up -d --remove-orphans +docker compose --env-file .env.test -p system-test down --volumes +# +# Note: cannot run as single docker compose command due to different build contexts +docker compose --env-file .env.test -p system-test up -d --build --remove-orphans +# Warning: Requires to be run from the project root cd .. && \ - docker compose --env-file test/.env.test -p test up -d --build && \ + docker compose --env-file test/.env.test -p system-test up -d --build && \ cd "${PACKAGE_DIR}/test" ./scripts/insert_test_data.sh -pip install "${PACKAGE_DIR}/pixl_core" "${PACKAGE_DIR}/cli" -pixl populate "${PACKAGE_DIR}/test/data/resources/omop" +pip install -e "${PACKAGE_DIR}/pixl_core" && pip install -e "${PACKAGE_DIR}/cli" +pixl populate "${PACKAGE_DIR}/test/resources/omop" pixl start sleep 65 # need to wait until the DICOM image is "stable" = 60s ./scripts/check_entry_in_pixl_anon.sh @@ -34,5 +37,4 @@ sleep 65 # need to wait until the DICOM image is "stable" = 60s ./scripts/check_max_storage_in_orthanc_raw.sh cd "${PACKAGE_DIR}" -docker compose -f docker-compose.yml -f ../docker-compose.yml -p test down -docker volume rm test_postgres-data test_orthanc-raw-data test_orthanc-anon-data +docker compose -f docker-compose.yml -f test/docker-compose.yml -p system-test down --volumes diff --git a/test/scripts/check_entry_in_orthanc_anon.sh b/test/scripts/check_entry_in_orthanc_anon.sh index 29bead1ac..253e981df 100755 --- a/test/scripts/check_entry_in_orthanc_anon.sh +++ b/test/scripts/check_entry_in_orthanc_anon.sh @@ -16,4 +16,4 @@ set -euxo pipefail # This could be much improved by having more realistic test data some of # which actually was persisted -docker logs test-orthanc-anon-1 2>&1 | grep "DICOM instance received" +docker logs system-test-orthanc-anon-1 2>&1 | grep "DICOM instance received" diff --git a/test/scripts/check_entry_in_pixl_anon.sh b/test/scripts/check_entry_in_pixl_anon.sh index 029251a0b..7051de342 100755 --- a/test/scripts/check_entry_in_pixl_anon.sh +++ b/test/scripts/check_entry_in_pixl_anon.sh @@ -15,6 +15,6 @@ set -euxo pipefail _sql_command="select * from emap_data.ehr_anon" -_result=$(docker exec -it test-postgres-1 /bin/bash -c \ +_result=$(docker exec system-test-postgres-1 /bin/bash -c \ "PGPASSWORD=pixl_db_password psql -U pixl_db_username -d pixl -c \"$_sql_command\"") -echo "$_result" | grep -q "1 row" +echo "$_result" | grep -q "6 row" diff --git a/test/scripts/check_max_storage_in_orthanc_raw.sh b/test/scripts/check_max_storage_in_orthanc_raw.sh index 8dea936c6..9b9a51716 100755 --- a/test/scripts/check_max_storage_in_orthanc_raw.sh +++ b/test/scripts/check_max_storage_in_orthanc_raw.sh @@ -17,5 +17,5 @@ set -euxo pipefail # This could be much improved by having more realistic test data some of # which actually was persisted source ./.env.test -docker logs test-orthanc-raw-1 2>&1 | grep "At most ${ORTHANC_RAW_MAXIMUM_STORAGE_SIZE}MB will be used for the storage area" +docker logs system-test-orthanc-raw-1 2>&1 | grep "At most ${ORTHANC_RAW_MAXIMUM_STORAGE_SIZE}MB will be used for the storage area" diff --git a/test/scripts/insert_test_data.sh b/test/scripts/insert_test_data.sh index 561452ce8..cae5edfe9 100755 --- a/test/scripts/insert_test_data.sh +++ b/test/scripts/insert_test_data.sh @@ -17,11 +17,18 @@ set -euxo pipefail SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) _sql_command=" -insert into star.mrn(mrn_id, mrn, research_opt_out) values (1234, 'patient_identifier', false); +insert into star.mrn(mrn_id, mrn, research_opt_out) values (1234, '12345678', false); +insert into star.mrn(mrn_id, mrn, research_opt_out) values (2345, '987654321', false); +insert into star.mrn(mrn_id, mrn, research_opt_out) values (3456, '5020765', false); insert into star.core_demographic(mrn_id, sex) values (1234, 'F'); +insert into star.core_demographic(mrn_id, sex) values (2345, 'F'); +insert into star.core_demographic(mrn_id, sex) values (3456, 'F'); " -docker exec -it test-fake-star-db /bin/bash -c "psql -U postgres -d emap -c \"$_sql_command\"" || true +docker exec -it system-test-fake-star-db /bin/bash -c "psql -U postgres -d emap -c \"$_sql_command\"" || true -# Uses an accession number of "123456789" -curl -X POST -u orthanc:orthanc http://localhost:8043/instances \ - --data-binary @"$SCRIPT_DIR/../data/test.dcm" +# Uses an accession number of "AA12345601" for MRN 987654321 +curl -X POST -u "orthanc:orthanc" "http://localhost:8043/instances" \ + --data-binary @"$SCRIPT_DIR/../resources/Dicom1.dcm" +# Uses an accession number of "AA12345605" for MRN 987654321 +curl -X POST -u "orthanc:orthanc" "http://localhost:8043/instances" \ + --data-binary @"$SCRIPT_DIR/../resources/Dicom2.dcm" \ No newline at end of file