diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index ed38782f9..2af954281 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -204,9 +204,7 @@ jobs: - name: Authenticate to Azure if: ${{ inputs.target == 'azure' }} - # There was a breaking change in azure/login, so we need to force the version - # https://github.com/Azure/login/issues/403 - uses: azure/login@v1.6.1 + uses: azure/login@v1 with: client-id: ${{ secrets.AZURE_CI_CLIENT }} tenant-id: ${{ secrets.AZURE_CI_TENANT }} @@ -229,12 +227,19 @@ jobs: . ./venv/bin/activate python -m pip install --upgrade pip + # Filter plugin dependencies, only install for the plugin being tested + # This prevents dependency issues in one plugin affecting all the others + - name: Select plugin dependencies + run: | + cd tracdap-runtime/python + sed -n '/BEGIN_PLUGIN ${{ inputs.target }}/, /END_PLUGIN ${{ inputs.target }}/p' requirements_plugins.txt > requirements_selected.txt + - name: Install dependencies run: | . ./venv/bin/activate cd tracdap-runtime/python pip install -r requirements.txt - pip install -r requirements_plugins.txt + pip install -r requirements_selected.txt - name: Protoc code generation run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8f64c95a8..3fff5cc02 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -366,6 +366,8 @@ jobs: PYTHON_TESTS: int_storage_aws*.py, JAVA_TESTS: int-storage } + # GCP does not support Python 3.12 yet, so run integration tests on 3.11 + # See https://github.com/googleapis/python-crc32c/issues/178 (there may be more dependencies affected) - { SERVICE: storage, TARGET: gcp, PROTOCOL: GCS, @@ -376,8 +378,11 @@ jobs: TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml', TRAC_SECRET_KEY: short-lived-secret, PYTHON_TESTS: int_storage_gcp*.py, + PYTHON_VERSION: "3.11", JAVA_TESTS: int-storage } + # Azure does not support Python 3.12 yet, so run integration tests on 3.11 + # See https://github.com/aio-libs/multidict/issues/887 (there may be more dependencies affected) - { SERVICE: storage, TARGET: azure, PROTOCOL: BLOB, @@ -388,6 +393,7 @@ jobs: TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml', TRAC_SECRET_KEY: short-lived-secret, PYTHON_TESTS: int_storage_azure*.py, + PYTHON_VERSION: "3.11", JAVA_TESTS: int-storage } uses: ./.github/workflows/integration-cloud.yaml diff --git a/tracdap-runtime/python/README.md b/tracdap-runtime/python/README.md index c677d13db..026433302 100644 --- a/tracdap-runtime/python/README.md +++ b/tracdap-runtime/python/README.md @@ -21,7 +21,7 @@ The TRAC runtime for Python has these requirements: * PySpark 3.0 up to 3.5 (optional) 3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark. - +As of January 2024, the Python libraries for GCP and Azure do not yet support Python 3.12. ## Installing the runtime diff --git a/tracdap-runtime/python/requirements_plugins.txt b/tracdap-runtime/python/requirements_plugins.txt index 0762828ae..ff6fa1313 100644 --- a/tracdap-runtime/python/requirements_plugins.txt +++ b/tracdap-runtime/python/requirements_plugins.txt @@ -9,11 +9,18 @@ # So e.g. to use the Azure libraries, Python 3.8 is required # We can still support older Python versions, so long as those plugins are not being used +# To control dependencies installed during CI, the BEGIN_PLUGIN and END_PLUGIN markers are used +# Running CI for a particular plugin will only install the dependencies between those markers +# This prevents dependency issues with one plugin breaking CI for all other plugins +# It also cuts down the setup time of integration jobs + # ---------------------------------------------------------------------------------------------------------------------- # AWS # ---------------------------------------------------------------------------------------------------------------------- +# BEGIN_PLUGIN aws + # AWS client libraries botocore == 1.29.156 @@ -23,10 +30,14 @@ boto3 == 1.26.156 # This is a separate library that depends on a very old version of botocore # It is not really possible to use them together and maintain compliance +# END_PLUGIN aws + # ---------------------------------------------------------------------------------------------------------------------- # GCP # ---------------------------------------------------------------------------------------------------------------------- +# BEGIN_PLUGIN gcp + # GCP client libraries google-auth == 2.20.0 @@ -37,12 +48,16 @@ google-cloud-storage == 2.9.0 gcsfs == 2023.6.0 +# END_PLUGIN gcp + # ---------------------------------------------------------------------------------------------------------------------- # Azure # ---------------------------------------------------------------------------------------------------------------------- -# AWS client libraries +# BEGIN_PLUGIN azure + +# Azure client libraries azure-core == 1.29.6 azure-identity == 1.15.0 @@ -55,3 +70,5 @@ adlfs == 2023.4.0 # azure-identity is still pulling in a compromised version of the cryptography package # force a compatible version with the required fix cryptography~=41.0.7 + +# END_PLUGIN azure