Fix / Cloud dependencies for Python 3.12

.github/workflows/integration-cloud.yaml

@@ -204,9 +204,7 @@ jobs:
 
       - name: Authenticate to Azure
         if: ${{ inputs.target == 'azure' }}
-        # There was a breaking change in azure/login, so we need to force the version
-        # https://github.com/Azure/login/issues/403
-        uses: azure/login@v1.6.1
+        uses: azure/login@v1
         with:
           client-id: ${{ secrets.AZURE_CI_CLIENT }}
           tenant-id: ${{ secrets.AZURE_CI_TENANT }}
@@ -229,12 +227,19 @@ jobs:
           . ./venv/bin/activate
           python -m pip install --upgrade pip
 
+      # Filter plugin dependencies, only install for the plugin being tested
+      # This prevents dependency issues in one plugin affecting all the others
+      - name: Select plugin dependencies
+        run: |
+          cd tracdap-runtime/python
+          sed -n '/BEGIN_PLUGIN ${{ inputs.target }}/, /END_PLUGIN ${{ inputs.target }}/p' requirements_plugins.txt > requirements_selected.txt
+
       - name: Install dependencies
         run: |
           . ./venv/bin/activate
           cd tracdap-runtime/python
           pip install -r requirements.txt
-          pip install -r requirements_plugins.txt
+          pip install -r requirements_selected.txt
 
       - name: Protoc code generation
         run: |

.github/workflows/integration.yml

@@ -366,6 +366,8 @@ jobs:
               PYTHON_TESTS: int_storage_aws*.py,
               JAVA_TESTS: int-storage }
 
+          # GCP does not support Python 3.12 yet, so run integration tests on 3.11
+          # See https://github.com/googleapis/python-crc32c/issues/178 (there may be more dependencies affected)
           - { SERVICE: storage,
               TARGET: gcp,
               PROTOCOL: GCS,
@@ -376,8 +378,11 @@ jobs:
               TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml',
               TRAC_SECRET_KEY: short-lived-secret,
               PYTHON_TESTS: int_storage_gcp*.py,
+              PYTHON_VERSION: "3.11",
               JAVA_TESTS: int-storage }
 
+          # Azure does not support Python 3.12 yet, so run integration tests on 3.11
+          # See https://github.com/aio-libs/multidict/issues/887 (there may be more dependencies affected)
           - { SERVICE: storage,
               TARGET: azure,
               PROTOCOL: BLOB,
@@ -388,6 +393,7 @@ jobs:
               TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml',
               TRAC_SECRET_KEY: short-lived-secret,
               PYTHON_TESTS: int_storage_azure*.py,
+              PYTHON_VERSION: "3.11",
               JAVA_TESTS: int-storage }
 
     uses: ./.github/workflows/integration-cloud.yaml

tracdap-runtime/python/README.md

@@ -21,7 +21,7 @@ The TRAC runtime for Python has these requirements:
 * PySpark 3.0 up to 3.5 (optional)
 
 3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
-
+As of January 2024, the Python libraries for GCP and Azure do not yet support Python 3.12.
 
 ## Installing the runtime
 

tracdap-runtime/python/requirements_plugins.txt

@@ -9,11 +9,18 @@
 # So e.g. to use the Azure libraries, Python 3.8 is required
 # We can still support older Python versions, so long as those plugins are not being used
 
+# To control dependencies installed during CI, the BEGIN_PLUGIN and END_PLUGIN markers are used
+# Running CI for a particular plugin will only install the dependencies between those markers
+# This prevents dependency issues with one plugin breaking CI for all other plugins
+# It also cuts down the setup time of integration jobs
+
 
 # ----------------------------------------------------------------------------------------------------------------------
 # AWS
 # ----------------------------------------------------------------------------------------------------------------------
 
+# BEGIN_PLUGIN aws
+
 # AWS client libraries
 
 botocore == 1.29.156
@@ -23,10 +30,14 @@ boto3 == 1.26.156
 # This is a separate library that depends on a very old version of botocore
 # It is not really possible to use them together and maintain compliance
 
+# END_PLUGIN aws
+
 # ----------------------------------------------------------------------------------------------------------------------
 # GCP
 # ----------------------------------------------------------------------------------------------------------------------
 
+# BEGIN_PLUGIN gcp
+
 # GCP client libraries
 
 google-auth == 2.20.0
@@ -37,12 +48,16 @@ google-cloud-storage == 2.9.0
 
 gcsfs == 2023.6.0
 
+# END_PLUGIN gcp
+
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Azure
 # ----------------------------------------------------------------------------------------------------------------------
 
-# AWS client libraries
+# BEGIN_PLUGIN azure
+
+# Azure client libraries
 
 azure-core == 1.29.6
 azure-identity == 1.15.0
@@ -55,3 +70,5 @@ adlfs == 2023.4.0
 # azure-identity is still pulling in a compromised version of the cryptography package
 # force a compatible version with the required fix
 cryptography~=41.0.7
+
+# END_PLUGIN azure