From 82f9de7ea1bff161f598654ff63a2332d849aa11 Mon Sep 17 00:00:00 2001 From: Francois Date: Mon, 10 Feb 2025 11:12:27 +0000 Subject: [PATCH 01/30] test new way of wiring up tests --- .github/workflows/core.yaml | 17 +++++++++----- Makefile | 7 +++--- bia-export/test/conftest.py | 8 ++++--- bia-test-data/bia_test_data/data_to_api.py | 26 +++++++++++++++++++--- docker-compose.yml | 14 +----------- 5 files changed, 43 insertions(+), 29 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 8a1ea1e3..5e21cfaf 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -44,11 +44,7 @@ jobs: matrix: python-version: ["3.11"] poetry-version: ["1.4.2"] - # NOTE - WE DO NOT TEST ON MAC/WINDOWS!! - # The current tests dockerize the test runner, so there is not much point on testing across platforms - # TODO: we want to change this to more closely follow how users actually run tests (setting up API first, then running tests) - # Ticket: https://app.clickup.com/t/8697kxtgx - os: [ubuntu-20.04] + os: [ubuntu-20.04, macos-latest, windows-latest] project: [ bia-export, @@ -66,5 +62,14 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - name: run makefile - run: make ${{matrix.project}}.test + run: make api-for-testing + - name: set up poetry in root of project + working-directory: ${{ matrix.project }} + run: poetry env use python + - name: install poetry + run: poetry install + - name: Check to see if poetry can build + run: poetry build + - name: Run pytest + run: poetry run pytest diff --git a/Makefile b/Makefile index c5285f0e..4c0ea98b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: api.version client.generate bia-export.test +.PHONY: api.version client.generate api-for-testing api.version: @echo $(shell grep '^version =' api/pyproject.toml | awk -F\" '{print $$2}') @@ -13,6 +13,5 @@ client.examples: docker compose --profile client_examples up --build --force-recreate --remove-orphans --abort-on-container-exit -# Note that the github CI is set up to expect Make commands for testing of the form: make ${{matrix.project}}.test -bia-export.test: - docker compose --profile export_test up --build --force-recreate --remove-orphans --abort-on-container-exit \ No newline at end of file +api-for-testing: + docker compose --profile bia-integrator-api up --build --force-recreate --remove-orphans --abort-on-container-exit \ No newline at end of file diff --git a/bia-export/test/conftest.py b/bia-export/test/conftest.py index 61293d1a..5b73aaa2 100644 --- a/bia-export/test/conftest.py +++ b/bia-export/test/conftest.py @@ -1,7 +1,6 @@ -from dotenv import load_dotenv from pathlib import Path import pytest -from bia_test_data.data_to_api import add_objects_to_api +from bia_test_data.data_to_api import add_objects_to_api, get_client_with_retries from bia_export.settings import Settings from pathlib import Path import json @@ -28,4 +27,7 @@ def data_in_api(): json_dict = json.load(object_file) object_list.append(json_dict) - add_objects_to_api(setttings.api_base_url, object_list) + + private_client = get_client_with_retries(setttings.api_base_url) + + add_objects_to_api(private_client, object_list) diff --git a/bia-test-data/bia_test_data/data_to_api.py b/bia-test-data/bia_test_data/data_to_api.py index d92bbd7f..544f6677 100644 --- a/bia-test-data/bia_test_data/data_to_api.py +++ b/bia-test-data/bia_test_data/data_to_api.py @@ -1,4 +1,6 @@ from pathlib import Path +from time import sleep +from urllib3.exceptions import MaxRetryError from dotenv import dotenv_values from bia_integrator_api.api import PrivateApi from bia_integrator_api import Configuration, ApiClient, exceptions @@ -41,6 +43,26 @@ def get_object_creation_client( return private_api +def get_client_with_retries( + api_base_url: str, wait_seconds: int = 5, max_retries: int = 5 +) -> PrivateApi: + private_client_retry_count = 0 + private_client = None + while not private_client and private_client_retry_count < max_retries: + try: + private_client = get_object_creation_client(api_base_url) + except MaxRetryError: + private_client_retry_count += 1 + sleep(wait_seconds) + + if private_client_retry_count == max_retries: + raise MaxRetryError( + "Attempted to create private client, but could not connect to API." + ) + + return private_client + + def calculate_dependency_chain_length( object_by_type: dict[str, dict[str, dict]] ) -> dict[str, int]: @@ -207,9 +229,7 @@ def order_object_for_api(object_list: list[dict]): return ordered_object_list -def add_objects_to_api(api_base_url, object_list: list[dict]): - private_client = get_object_creation_client(api_base_url) - +def add_objects_to_api(private_client, object_list: list[dict]): ordered_object_list = order_object_for_api(object_list) for bia_object_dict in ordered_object_list: diff --git a/docker-compose.yml b/docker-compose.yml index 8f3c69fc..df19049c 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,16 +59,4 @@ services: volumes: - ./clients/python:/mnt/share profiles: - - client_examples - export-test: - depends_on: - bia-integrator-api: - condition: service_healthy - environment: - - API_BASE_URL=http://api:8080 - container_name: export_test - image: bia-integrator-api - command: bash -c "echo $PWD && poetry install && poetry run pytest -vv" - working_dir: /bia-integrator/bia-export - profiles: - - export_test \ No newline at end of file + - client_examples \ No newline at end of file From ab37ff062eacdc123da007345052fb87ada323b4 Mon Sep 17 00:00:00 2001 From: Francois Date: Mon, 10 Feb 2025 11:27:29 +0000 Subject: [PATCH 02/30] update ci commands --- .github/workflows/core.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 5e21cfaf..ec2672dc 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -52,17 +52,15 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 # Set up Docker Buildx for building and pushing Docker images - - name: Cache Docker layers - uses: actions/cache@v4 + - uses: actions/setup-python@v4 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: run makefile - run: make api-for-testing + python-version: ${{ matrix.python-version }} + - name: Run poetry image + uses: abatilo/actions-poetry@v2 + with: + poetry-version: ${{ matrix.poetry-version }} + - name: run api compose makefile + run: docker compose --profile bia-integrator-api up --build --force-recreate --remove-orphans --abort-on-container-exit - name: set up poetry in root of project working-directory: ${{ matrix.project }} run: poetry env use python From 279bb04cfd04ac931a4d505923dac0b4f67d5640 Mon Sep 17 00:00:00 2001 From: Francois Date: Mon, 10 Feb 2025 11:38:20 +0000 Subject: [PATCH 03/30] update docker workflow --- .github/workflows/core.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index ec2672dc..516cd249 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -59,8 +59,16 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: ${{ matrix.poetry-version }} - - name: run api compose makefile - run: docker compose --profile bia-integrator-api up --build --force-recreate --remove-orphans --abort-on-container-exit + - name: Set up Docker on macOS + if: runner.os == 'macOS' + run: | + brew install docker colima + colima start + - name: Verify Docker installation + run: docker --version + - name: Docker Compose Up + run: | + docker compose up -d --build - name: set up poetry in root of project working-directory: ${{ matrix.project }} run: poetry env use python @@ -70,4 +78,9 @@ jobs: run: poetry build - name: Run pytest run: poetry run pytest + # Always cleanup - even for cancelled jobs + - name: Docker Compose Down + run: | + docker compose down + if: ${{ always() }} From 751ad13b4db289c0336eeaa621b7664cbf3698c2 Mon Sep 17 00:00:00 2001 From: Francois Date: Mon, 10 Feb 2025 11:57:41 +0000 Subject: [PATCH 04/30] change mac docker installation --- .github/workflows/core.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 516cd249..58daec76 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -50,6 +50,9 @@ jobs: bia-export, ] runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ matrix.project }} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -62,18 +65,16 @@ jobs: - name: Set up Docker on macOS if: runner.os == 'macOS' run: | - brew install docker colima - colima start - - name: Verify Docker installation + brew install docker + brew services start docker + - name: Verify Docker is installed run: docker --version - name: Docker Compose Up + run: docker compose up -d --build + - name: set up poetry run: | - docker compose up -d --build - - name: set up poetry in root of project - working-directory: ${{ matrix.project }} - run: poetry env use python - - name: install poetry - run: poetry install + poetry env use python + poetry install - name: Check to see if poetry can build run: poetry build - name: Run pytest From b10456e3ff6cfb9eef3983c24205febfa6fd6ff1 Mon Sep 17 00:00:00 2001 From: Francois Date: Mon, 10 Feb 2025 16:25:38 +0000 Subject: [PATCH 05/30] remove windows workflow --- .github/workflows/core.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 58daec76..d197d48d 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -44,7 +44,8 @@ jobs: matrix: python-version: ["3.11"] poetry-version: ["1.4.2"] - os: [ubuntu-20.04, macos-latest, windows-latest] + # Note we do not test on windows + os: [ubuntu-20.04, macos-latest] project: [ bia-export, From 9cce4cbec94e12e1f36451b002fa2069637a9f86 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:29:21 +0000 Subject: [PATCH 06/30] trying to use colima for docker --- .github/workflows/core.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index d197d48d..ae1611ca 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -66,8 +66,9 @@ jobs: - name: Set up Docker on macOS if: runner.os == 'macOS' run: | - brew install docker - brew services start docker + brew install docker colima + colima start --cpu 2 --memory 4 --disk 10 + docker context use colima - name: Verify Docker is installed run: docker --version - name: Docker Compose Up From 6b7fb01de5b0f57c5034aec15ed8aec55ecb6651 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:43:51 +0000 Subject: [PATCH 07/30] try using python-on-whales to start docker container --- .github/workflows/core.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index ae1611ca..ad340fb6 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -63,16 +63,16 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: ${{ matrix.poetry-version }} - - name: Set up Docker on macOS + - name: Use python-on-whales to start api in docker container on mac if: runner.os == 'macOS' run: | - brew install docker colima - colima start --cpu 2 --memory 4 --disk 10 - docker context use colima - - name: Verify Docker is installed - run: docker --version - - name: Docker Compose Up - run: docker compose up -d --build + python -m pip install --upgrade pip + pip install python-on-whales + python -c "from python_on_whales import docker; print(docker.context.inspect())" + ython -c "from python_on_whales import docker_compose; docker_compose.up(build=True, force_recreate=True, remove_orphans=True, abort_on_container_exit=True, profiles=['bia-integrator-api'])" + - name: Run docker normally if on linux + if: runner.os == 'Linux' + run: docker compose up -d --build - name: set up poetry run: | poetry env use python From 9324f3be0a6fb00b4eea9e5dff14bff0628531d8 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:47:04 +0000 Subject: [PATCH 08/30] try installing docker first --- .github/workflows/core.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index ad340fb6..fc127bc9 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -66,6 +66,7 @@ jobs: - name: Use python-on-whales to start api in docker container on mac if: runner.os == 'macOS' run: | + brew install docker python -m pip install --upgrade pip pip install python-on-whales python -c "from python_on_whales import docker; print(docker.context.inspect())" From 63986f66d1839e9ae2a6fb158484430633453fb2 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:49:13 +0000 Subject: [PATCH 09/30] fix typo --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index fc127bc9..e546b4fe 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -70,7 +70,7 @@ jobs: python -m pip install --upgrade pip pip install python-on-whales python -c "from python_on_whales import docker; print(docker.context.inspect())" - ython -c "from python_on_whales import docker_compose; docker_compose.up(build=True, force_recreate=True, remove_orphans=True, abort_on_container_exit=True, profiles=['bia-integrator-api'])" + python -c "from python_on_whales import docker_compose; docker_compose.up(build=True, force_recreate=True, remove_orphans=True, abort_on_container_exit=True, profiles=['bia-integrator-api'])" - name: Run docker normally if on linux if: runner.os == 'Linux' run: docker compose up -d --build From beb82038b1be0b70ee66a5be2861776e912d45ac Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:54:23 +0000 Subject: [PATCH 10/30] change python command --- .github/workflows/core.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index e546b4fe..c729340f 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -69,8 +69,7 @@ jobs: brew install docker python -m pip install --upgrade pip pip install python-on-whales - python -c "from python_on_whales import docker; print(docker.context.inspect())" - python -c "from python_on_whales import docker_compose; docker_compose.up(build=True, force_recreate=True, remove_orphans=True, abort_on_container_exit=True, profiles=['bia-integrator-api'])" + python -c "from python_on_whales import docker; docker.compose.up(build=True, force_recreate=True, remove_orphans=True, detatch=True, profiles=['bia-integrator-api'])" - name: Run docker normally if on linux if: runner.os == 'Linux' run: docker compose up -d --build From 4a9fa3bed1a4dbcb8ad29dfdaabb1c87bee9b33a Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:55:20 +0000 Subject: [PATCH 11/30] fix typo --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index c729340f..462c64cf 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -69,7 +69,7 @@ jobs: brew install docker python -m pip install --upgrade pip pip install python-on-whales - python -c "from python_on_whales import docker; docker.compose.up(build=True, force_recreate=True, remove_orphans=True, detatch=True, profiles=['bia-integrator-api'])" + python -c "from python_on_whales import docker; docker.compose.up(build=True, force_recreate=True, remove_orphans=True, detach=True, profiles=['bia-integrator-api'])" - name: Run docker normally if on linux if: runner.os == 'Linux' run: docker compose up -d --build From 02b919f751067c60a5f35e1b66442c9e7a4a0023 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 10:59:15 +0000 Subject: [PATCH 12/30] make mac command equivalent to linux --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 462c64cf..023a5d9e 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -69,7 +69,7 @@ jobs: brew install docker python -m pip install --upgrade pip pip install python-on-whales - python -c "from python_on_whales import docker; docker.compose.up(build=True, force_recreate=True, remove_orphans=True, detach=True, profiles=['bia-integrator-api'])" + python -c "from python_on_whales import docker; docker.compose.up(build=True, detach=True)" - name: Run docker normally if on linux if: runner.os == 'Linux' run: docker compose up -d --build From 44e0818a325d64b54827d530122111d227c122af Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 13:19:03 +0000 Subject: [PATCH 13/30] add to github path --- .github/workflows/core.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 023a5d9e..c343f46b 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -67,6 +67,7 @@ jobs: if: runner.os == 'macOS' run: | brew install docker + echo "$(brew --prefix)/bin" >> $GITHUB_PATH python -m pip install --upgrade pip pip install python-on-whales python -c "from python_on_whales import docker; docker.compose.up(build=True, detach=True)" From 62afa2b85fd9930f31ccd699b5fcdb1e35fef674 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 13:26:11 +0000 Subject: [PATCH 14/30] update python commands --- .github/workflows/core.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index c343f46b..4fe0510c 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -69,8 +69,8 @@ jobs: brew install docker echo "$(brew --prefix)/bin" >> $GITHUB_PATH python -m pip install --upgrade pip - pip install python-on-whales - python -c "from python_on_whales import docker; docker.compose.up(build=True, detach=True)" + pip3 install python-on-whales + python3 -c "from python_on_whales import docker; docker.compose.up(detach=True, build=True)" - name: Run docker normally if on linux if: runner.os == 'Linux' run: docker compose up -d --build From 7c37f52029da56ab13b2b8f2507b0457005bec36 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 13:29:50 +0000 Subject: [PATCH 15/30] add docker-compose plugin to brew install --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 4fe0510c..219582a3 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -66,7 +66,7 @@ jobs: - name: Use python-on-whales to start api in docker container on mac if: runner.os == 'macOS' run: | - brew install docker + brew install docker docker-compose echo "$(brew --prefix)/bin" >> $GITHUB_PATH python -m pip install --upgrade pip pip3 install python-on-whales From ef87af0aa25e49bb27c97d70105791ab3b1d6a88 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 13:52:50 +0000 Subject: [PATCH 16/30] try using colima --- .github/workflows/core.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 219582a3..c95cf0da 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -66,13 +66,10 @@ jobs: - name: Use python-on-whales to start api in docker container on mac if: runner.os == 'macOS' run: | - brew install docker docker-compose + brew install docker colima echo "$(brew --prefix)/bin" >> $GITHUB_PATH - python -m pip install --upgrade pip - pip3 install python-on-whales - python3 -c "from python_on_whales import docker; docker.compose.up(detach=True, build=True)" + colima start - name: Run docker normally if on linux - if: runner.os == 'Linux' run: docker compose up -d --build - name: set up poetry run: | From 18b721f928ff7330c2597cbf47a2fb72dec23734 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:11:05 +0000 Subject: [PATCH 17/30] try macos 13 runners --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index c95cf0da..e30ade27 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -8,7 +8,7 @@ jobs: matrix: python-version: ["3.11"] poetry-version: ["1.4.2"] - os: [ubuntu-20.04, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-13, windows-latest] project: [ bia-ingest, From 3f6012dd83845f36356624e3944b32d91cb94035 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:33:19 +0000 Subject: [PATCH 18/30] switch to just testing on linux since mac runners do not support nested virtualisation --- .github/workflows/core.yaml | 15 +++++---------- Makefile | 4 ---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index e30ade27..1d0cd6d6 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -38,14 +38,15 @@ jobs: run: poetry build - name: Run pytest run: poetry run pytest - ci-make: + ci-test-with-api: strategy: fail-fast: false matrix: python-version: ["3.11"] poetry-version: ["1.4.2"] - # Note we do not test on windows - os: [ubuntu-20.04, macos-latest] + # Note we do not test on windows or mac. Currently, github mac runners do not support nested virtualization, so we cannot run docker on them (since mac docker virtualises linux) + # See https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#limitations-for-arm64-macos-runners + os: [ubuntu-20.04] project: [ bia-export, @@ -63,13 +64,7 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: ${{ matrix.poetry-version }} - - name: Use python-on-whales to start api in docker container on mac - if: runner.os == 'macOS' - run: | - brew install docker colima - echo "$(brew --prefix)/bin" >> $GITHUB_PATH - colima start - - name: Run docker normally if on linux + - name: Run docker run: docker compose up -d --build - name: set up poetry run: | diff --git a/Makefile b/Makefile index 4c0ea98b..291e03d4 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,3 @@ client.generate: client.examples: docker compose --profile client_examples up --build --force-recreate --remove-orphans --abort-on-container-exit - - -api-for-testing: - docker compose --profile bia-integrator-api up --build --force-recreate --remove-orphans --abort-on-container-exit \ No newline at end of file From 77b77db4482215ba1618f32d0b62266fea3b1455 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:35:19 +0000 Subject: [PATCH 19/30] fix mac os to latest --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 1d0cd6d6..8bebd3b9 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -8,7 +8,7 @@ jobs: matrix: python-version: ["3.11"] poetry-version: ["1.4.2"] - os: [ubuntu-20.04, macos-13, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] project: [ bia-ingest, From 8a4c8285c41eb13b5c3fd5e3b8e7702aa181d69d Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:38:12 +0000 Subject: [PATCH 20/30] try colima on macos-13 --- .github/workflows/core.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 8bebd3b9..a1ce9532 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -46,7 +46,7 @@ jobs: poetry-version: ["1.4.2"] # Note we do not test on windows or mac. Currently, github mac runners do not support nested virtualization, so we cannot run docker on them (since mac docker virtualises linux) # See https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#limitations-for-arm64-macos-runners - os: [ubuntu-20.04] + os: [ubuntu-20.04, mac-os-13] project: [ bia-export, @@ -64,6 +64,12 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: ${{ matrix.poetry-version }} + - name: mac needs colima + if: runner.os == 'macOS' + run: | + brew install docker colima + echo "$(brew --prefix)/bin" >> $GITHUB_PATH + colima start - name: Run docker run: docker compose up -d --build - name: set up poetry From bfee5a2066b1093f2d288270104ccb5890800806 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:40:45 +0000 Subject: [PATCH 21/30] fix macos-13 --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index a1ce9532..ce448026 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -46,7 +46,7 @@ jobs: poetry-version: ["1.4.2"] # Note we do not test on windows or mac. Currently, github mac runners do not support nested virtualization, so we cannot run docker on them (since mac docker virtualises linux) # See https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#limitations-for-arm64-macos-runners - os: [ubuntu-20.04, mac-os-13] + os: [ubuntu-20.04, macos-13] project: [ bia-export, From f19a94cfe00cfba4d0f8f6a463668b75789c6532 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:47:56 +0000 Subject: [PATCH 22/30] try --detach --- .github/workflows/core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index ce448026..1c238e7a 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -71,7 +71,7 @@ jobs: echo "$(brew --prefix)/bin" >> $GITHUB_PATH colima start - name: Run docker - run: docker compose up -d --build + run: docker compose up --detach --build - name: set up poetry run: | poetry env use python From 1bb926c09baa4f2cb25fdcc5e384d4aa6500d72c Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 14:57:53 +0000 Subject: [PATCH 23/30] give up on running docker on mac --- .github/workflows/core.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 1c238e7a..8bebd3b9 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -46,7 +46,7 @@ jobs: poetry-version: ["1.4.2"] # Note we do not test on windows or mac. Currently, github mac runners do not support nested virtualization, so we cannot run docker on them (since mac docker virtualises linux) # See https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#limitations-for-arm64-macos-runners - os: [ubuntu-20.04, macos-13] + os: [ubuntu-20.04] project: [ bia-export, @@ -64,14 +64,8 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: ${{ matrix.poetry-version }} - - name: mac needs colima - if: runner.os == 'macOS' - run: | - brew install docker colima - echo "$(brew --prefix)/bin" >> $GITHUB_PATH - colima start - name: Run docker - run: docker compose up --detach --build + run: docker compose up -d --build - name: set up poetry run: | poetry env use python From dd1d91452822ec6887d5d96406e72c02887b0fbb Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 15:37:54 +0000 Subject: [PATCH 24/30] added test for all export option --- bia-export/bia_export/cli.py | 24 +++++++++---- bia-export/test/conftest.py | 3 +- bia-export/test/test_api_export.py | 56 +++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/bia-export/bia_export/cli.py b/bia-export/bia_export/cli.py index 96ba3369..26be6caf 100644 --- a/bia-export/bia_export/cli.py +++ b/bia-export/bia_export/cli.py @@ -27,6 +27,11 @@ app.add_typer(website, name="website") +DEFAULT_WEBSITE_STUDY_FILE_NAME = "bia-study-metadata.json" +DEFAULT_WEBSITE_IMAGE_FILE_NAME = "bia-image-metadata.json" +DEFAULT_WEBSITE_DATASET_FOR_IMAGE_FILE_NAME = "bia-dataset-metadata-for-images.json" + + @website.command("all") def generate_all( id_list: Annotated[ @@ -47,6 +52,13 @@ def generate_all( "-c", ), ] = None, + output_directory: Annotated[ + Optional[Path], + typer.Option( + "--out_dir", + "-o", + ), + ] = None, ): settings = Settings() @@ -54,11 +66,11 @@ def generate_all( id_list = get_study_ids(root_directory) logger.info("Exporting study pages") - website_study(id_list=id_list, root_directory=root_directory, cache=cache) + website_study(id_list=id_list, root_directory=root_directory, cache=cache, output_filename=(output_directory / DEFAULT_WEBSITE_STUDY_FILE_NAME if output_directory else None)) logger.info("Exporting image pages") - website_image(id_list=id_list, root_directory=root_directory) + website_image(id_list=id_list, root_directory=root_directory, output_filename=(output_directory / DEFAULT_WEBSITE_IMAGE_FILE_NAME if output_directory else None)) logger.info("Exporting datasets for study pages") - datasets_for_website_image(id_list=id_list, root_directory=root_directory) + datasets_for_website_image(id_list=id_list, root_directory=root_directory, output_filename=(output_directory / DEFAULT_WEBSITE_DATASET_FOR_IMAGE_FILE_NAME if output_directory else None)) @website.command("study") @@ -72,7 +84,7 @@ def website_study( "--out_file", "-o", ), - ] = Path("bia-study-metadata.json"), + ] = Path(DEFAULT_WEBSITE_STUDY_FILE_NAME), root_directory: Annotated[ Optional[Path], typer.Option( @@ -117,7 +129,7 @@ def website_image( "--out_file", "-o", ), - ] = Path("bia-image-metadata.json"), + ] = Path(DEFAULT_WEBSITE_IMAGE_FILE_NAME), root_directory: Annotated[ Optional[Path], typer.Option( @@ -151,7 +163,7 @@ def datasets_for_website_image( "--out_file", "-o", ), - ] = Path("bia-dataset-metadata-for-images.json"), + ] = Path(DEFAULT_WEBSITE_DATASET_FOR_IMAGE_FILE_NAME), root_directory: Annotated[ Optional[Path], typer.Option( diff --git a/bia-export/test/conftest.py b/bia-export/test/conftest.py index 5b73aaa2..15336369 100644 --- a/bia-export/test/conftest.py +++ b/bia-export/test/conftest.py @@ -9,8 +9,7 @@ def pytest_configure(config: pytest.Config): - if not os.environ.get("API_BASE_URL", None): - os.environ.setdefault("API_BASE_URL", "http://localhost:8080") + os.environ.setdefault("API_BASE_URL", "http://localhost:8080") @pytest.fixture(scope="session") diff --git a/bia-export/test/test_api_export.py b/bia-export/test/test_api_export.py index da9f03ea..aece56ed 100644 --- a/bia-export/test/test_api_export.py +++ b/bia-export/test/test_api_export.py @@ -1,11 +1,65 @@ from typer.testing import CliRunner from pathlib import Path -from bia_export.cli import app +from bia_export.cli import ( + app, + DEFAULT_WEBSITE_STUDY_FILE_NAME, + DEFAULT_WEBSITE_IMAGE_FILE_NAME, + DEFAULT_WEBSITE_DATASET_FOR_IMAGE_FILE_NAME, +) import json +import os runner = CliRunner() +# Note that pytest fixture data_in_api is unused in test code, +# but requested in order to guarentee that data is in the test api before running the test +def test_cli_export_export_all_data_contains_at_least_expected_objects( + tmp_path: Path, data_in_api +): + + outpath = tmp_path.resolve() + + result = runner.invoke( + app, + [ + "website", + "all", + "-o", + outpath, + ], + ) + + assert result.exit_code == 0 + assert len(os.listdir(tmp_path)) == 3 + + def check_file_contains_expected_object(outfile: Path, expected_output: Path): + with open(outfile, "r") as f: + json_result = json.load(f) + + with open(expected_output) as f: + json_expected = json.load(f) + + for key, value in json_expected.items(): + assert key in json_result + assert value == json_result[key] + + check_file_contains_expected_object( + outpath.joinpath(DEFAULT_WEBSITE_STUDY_FILE_NAME), + Path(__file__).parent.joinpath("output_data/bia-study-metadata.json"), + ) + + check_file_contains_expected_object( + outpath.joinpath(DEFAULT_WEBSITE_IMAGE_FILE_NAME), + Path(__file__).parent.joinpath("output_data/bia-image-metadata.json"), + ) + + check_file_contains_expected_object( + outpath.joinpath(DEFAULT_WEBSITE_DATASET_FOR_IMAGE_FILE_NAME), + Path(__file__).parent.joinpath("output_data/bia-dataset-metadata-for-images.json"), + ) + + # Note that pytest fixture data_in_api is unused in test code, # but requested in order to guarentee that data is in the test api before running the test def test_cli_export_website_studies(tmp_path: Path, data_in_api): From ffd5011ff7a4cbca45a60456cb60da4c293c95ab Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 12 Feb 2025 17:41:19 +0000 Subject: [PATCH 25/30] added order test --- bia-export/test/conftest.py | 52 ++++++++++++++++++++++++++++++ bia-export/test/test_api_export.py | 26 +++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/bia-export/test/conftest.py b/bia-export/test/conftest.py index 15336369..fb03a91f 100644 --- a/bia-export/test/conftest.py +++ b/bia-export/test/conftest.py @@ -2,6 +2,7 @@ import pytest from bia_test_data.data_to_api import add_objects_to_api, get_client_with_retries from bia_export.settings import Settings +from bia_shared_datamodels.uuid_creation import create_study_uuid from pathlib import Path import json import os @@ -30,3 +31,54 @@ def data_in_api(): private_client = get_client_with_retries(setttings.api_base_url) add_objects_to_api(private_client, object_list) + + + + +@pytest.fixture(scope="session") +def api_studies_in_expected_order(): + setttings = Settings() + + base_study = Path(__file__).parent / "input_data" / "study" / "S-BIADTEST" / "a2fdbd58-ee11-4cd9-bc6a-f3d3da7fff71.json" + + with open(base_study, "r") as object_file: + base_study_dict: dict = json.load(object_file) + + object_list = [] + + study_1 = base_study_dict.copy() | { + "accession_id": "S-BIADTEST1", + "uuid": str(create_study_uuid("S-BIADTEST111")), + "release_date": "2024-01-01" + } + object_list.append(study_1) + + study_2 = base_study_dict.copy() | { + "accession_id": "S-BIADTEST22", + "uuid": str(create_study_uuid("S-BIADTEST22")), + "release_date": "2024-01-01" + } + object_list.append(study_2) + + study_3 = base_study_dict.copy() | { + "accession_id": "S-BIADTEST333", + "uuid": str(create_study_uuid("S-BIADTEST333")), + "release_date": "2024-01-03" + } + object_list.append(study_3) + + + study_4 = base_study_dict.copy() | { + "accession_id": "S-BIADTEST4444", + "uuid": str(create_study_uuid("S-BIADTEST4444")), + "release_date": "2024-01-02" + } + object_list.append(study_4) + + private_client = get_client_with_retries(setttings.api_base_url) + + add_objects_to_api(private_client, object_list) + + expected_order_of_studies = [study_3, study_4, study_2, study_1] + + return expected_order_of_studies diff --git a/bia-export/test/test_api_export.py b/bia-export/test/test_api_export.py index aece56ed..725cfd01 100644 --- a/bia-export/test/test_api_export.py +++ b/bia-export/test/test_api_export.py @@ -1,5 +1,6 @@ from typer.testing import CliRunner from pathlib import Path +import pytest from bia_export.cli import ( app, DEFAULT_WEBSITE_STUDY_FILE_NAME, @@ -133,3 +134,28 @@ def test_cli_export_dataset_for_website_images(tmp_path: Path, data_in_api): json_expected = json.load(f) assert json_result == json_expected + + + +def test_cli_export_study_ordering(tmp_path: Path, api_studies_in_expected_order: list[dict]): + outfile = tmp_path.joinpath("bia-dataset-metadata.json").resolve() + + result = runner.invoke( + app, + [ + "website", + "study", + "-o", + outfile, + ], + ) + + assert result.exit_code == 0 + + expected_study_acc_id_order = [study["accession_id"] for study in api_studies_in_expected_order] + + with open(outfile, "r") as f: + json_result: dict = json.load(f) + + it = iter(json_result.keys()) + assert all(acc_id in it for acc_id in expected_study_acc_id_order) \ No newline at end of file From 309a66a3e2c7b7c3c912ef3e6fe5e44d366222e4 Mon Sep 17 00:00:00 2001 From: Francois Date: Thu, 13 Feb 2025 12:19:21 +0000 Subject: [PATCH 26/30] clean up leftovers from previous commits, and use --wait to wait for api deployment --- .github/workflows/core.yaml | 4 +--- Makefile | 2 +- bia-export/test/conftest.py | 6 +++--- bia-test-data/bia_test_data/data_to_api.py | 20 -------------------- 4 files changed, 5 insertions(+), 27 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 8bebd3b9..c7d8d0d7 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -65,13 +65,11 @@ jobs: with: poetry-version: ${{ matrix.poetry-version }} - name: Run docker - run: docker compose up -d --build + run: docker compose up -d --build --wait - name: set up poetry run: | poetry env use python poetry install - - name: Check to see if poetry can build - run: poetry build - name: Run pytest run: poetry run pytest # Always cleanup - even for cancelled jobs diff --git a/Makefile b/Makefile index 291e03d4..92983874 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: api.version client.generate api-for-testing +.PHONY: api.version client.generate api.version: @echo $(shell grep '^version =' api/pyproject.toml | awk -F\" '{print $$2}') diff --git a/bia-export/test/conftest.py b/bia-export/test/conftest.py index fb03a91f..465f3567 100644 --- a/bia-export/test/conftest.py +++ b/bia-export/test/conftest.py @@ -1,6 +1,6 @@ from pathlib import Path import pytest -from bia_test_data.data_to_api import add_objects_to_api, get_client_with_retries +from bia_test_data.data_to_api import add_objects_to_api, get_object_creation_client from bia_export.settings import Settings from bia_shared_datamodels.uuid_creation import create_study_uuid from pathlib import Path @@ -28,7 +28,7 @@ def data_in_api(): object_list.append(json_dict) - private_client = get_client_with_retries(setttings.api_base_url) + private_client = get_object_creation_client(setttings.api_base_url) add_objects_to_api(private_client, object_list) @@ -75,7 +75,7 @@ def api_studies_in_expected_order(): } object_list.append(study_4) - private_client = get_client_with_retries(setttings.api_base_url) + private_client = get_object_creation_client(setttings.api_base_url) add_objects_to_api(private_client, object_list) diff --git a/bia-test-data/bia_test_data/data_to_api.py b/bia-test-data/bia_test_data/data_to_api.py index 544f6677..c852bcf8 100644 --- a/bia-test-data/bia_test_data/data_to_api.py +++ b/bia-test-data/bia_test_data/data_to_api.py @@ -43,26 +43,6 @@ def get_object_creation_client( return private_api -def get_client_with_retries( - api_base_url: str, wait_seconds: int = 5, max_retries: int = 5 -) -> PrivateApi: - private_client_retry_count = 0 - private_client = None - while not private_client and private_client_retry_count < max_retries: - try: - private_client = get_object_creation_client(api_base_url) - except MaxRetryError: - private_client_retry_count += 1 - sleep(wait_seconds) - - if private_client_retry_count == max_retries: - raise MaxRetryError( - "Attempted to create private client, but could not connect to API." - ) - - return private_client - - def calculate_dependency_chain_length( object_by_type: dict[str, dict[str, dict]] ) -> dict[str, int]: From 18334bc18a79a8e0ca98f977c5fcb04a1f499c9d Mon Sep 17 00:00:00 2001 From: Francois Date: Fri, 14 Feb 2025 10:22:33 +0000 Subject: [PATCH 27/30] change to use makefiles --- .github/workflows/api.yaml | 4 ++-- Makefile | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 2fe35f83..81f023cb 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -36,7 +36,7 @@ jobs: ${{ runner.os }}-buildx- - name: Docker Compose Up run: | - docker compose up -d --build + make api.up - name: Run tests # api moved to api/ , src left in as a reference as we transition. All tests there are failing and that directory will be deleted anyway run: | @@ -47,5 +47,5 @@ jobs: # Always cleanup - even for cancelled jobs - name: Docker Compose Down run: | - docker compose down + make api.down if: ${{ always() }} \ No newline at end of file diff --git a/Makefile b/Makefile index 92983874..d14773db 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: api.version client.generate +.PHONY: api.version client.generate api.up api.down api.version: @echo $(shell grep '^version =' api/pyproject.toml | awk -F\" '{print $$2}') @@ -11,3 +11,9 @@ client.generate: client.examples: docker compose --profile client_examples up --build --force-recreate --remove-orphans --abort-on-container-exit + +api.up: + docker compose up -d --build --wait + +api.down: + docker compose down From b2f7da0bf54c42a0497dbb3a29ff0620214e2b09 Mon Sep 17 00:00:00 2001 From: Francois Date: Fri, 14 Feb 2025 10:26:45 +0000 Subject: [PATCH 28/30] whitespace change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d14773db..4c7549d6 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ client.examples: docker compose --profile client_examples up --build --force-recreate --remove-orphans --abort-on-container-exit api.up: - docker compose up -d --build --wait + docker compose up -d --build --wait api.down: docker compose down From 07bb29fd61007994bbcda078ab64c508611ea360 Mon Sep 17 00:00:00 2001 From: Francois Date: Fri, 14 Feb 2025 10:32:51 +0000 Subject: [PATCH 29/30] fixes --- .github/workflows/api.yaml | 4 ++-- .github/workflows/core.yaml | 5 ++--- Makefile | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 81f023cb..2fe35f83 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -36,7 +36,7 @@ jobs: ${{ runner.os }}-buildx- - name: Docker Compose Up run: | - make api.up + docker compose up -d --build - name: Run tests # api moved to api/ , src left in as a reference as we transition. All tests there are failing and that directory will be deleted anyway run: | @@ -47,5 +47,5 @@ jobs: # Always cleanup - even for cancelled jobs - name: Docker Compose Down run: | - make api.down + docker compose down if: ${{ always() }} \ No newline at end of file diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index c7d8d0d7..3429e1c9 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -65,7 +65,7 @@ jobs: with: poetry-version: ${{ matrix.poetry-version }} - name: Run docker - run: docker compose up -d --build --wait + run: make api.up - name: set up poetry run: | poetry env use python @@ -74,7 +74,6 @@ jobs: run: poetry run pytest # Always cleanup - even for cancelled jobs - name: Docker Compose Down - run: | - docker compose down + run: make api.down if: ${{ always() }} diff --git a/Makefile b/Makefile index 4c7549d6..8f145fcb 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ client.examples: docker compose --profile client_examples up --build --force-recreate --remove-orphans --abort-on-container-exit api.up: - docker compose up -d --build --wait + docker compose up -d --build --wait api.down: docker compose down From dc4e65c43e35901a7d0eeffafecc87125b0fa350 Mon Sep 17 00:00:00 2001 From: Francois Date: Fri, 14 Feb 2025 10:45:16 +0000 Subject: [PATCH 30/30] update make commands --- .github/workflows/core.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 3429e1c9..ca5b8c89 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -65,7 +65,7 @@ jobs: with: poetry-version: ${{ matrix.poetry-version }} - name: Run docker - run: make api.up + run: make -C ../ api.up - name: set up poetry run: | poetry env use python @@ -74,6 +74,6 @@ jobs: run: poetry run pytest # Always cleanup - even for cancelled jobs - name: Docker Compose Down - run: make api.down + run: make -C ../ api.down if: ${{ always() }}