[DPE-2788] update data-platform-workflows (#308)

* Initial switch

* Add missing integration deps

* Remove schedule

* No excludes

* Bump libs and actions

* Increase upgrade timeout

Author: dragomirp

.github/renovate.json5

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["github>canonical/data-platform//renovate_presets/charm.json5"],
+  "reviewers": ["dragomirp", "marceloneppel", "taurus-forever"],
+  "packageRules": [
+    // Later rules override earlier rules
+  ],
+  "regexManagers": [
+    {
+      "fileMatch": ["(^|/)([\\w-]*)charmcraft\\.ya?ml$"],
+      "matchStrings": ["- (?<depName>.*?)(?:\\[.*?\\])?==(?<currentValue>.*?) +# renovate"],
+      "datasourceTemplate": "pypi",
+      "versioningTemplate": "loose"
+    }
+  ]
+}

.github/workflows/ci.yaml

@@ -22,45 +22,38 @@ on:
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
-      - name: Run linters
-        run: tox run -e lint
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v5.1.2
 
   unit-test:
-    name: Unit tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
+    name: Unit test charm
     strategy:
       matrix:
         juju-version:
           - "juju2"
           - "juju3"
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
+        uses: actions/checkout@v4
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Run tests
         run: tox run -e unit-${{ matrix.juju-version }}
 
   build:
-    name: Build charms
-    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2
+    name: Build charm
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v5.1.2
+    permissions:
+      actions: write  # Needed to manage GitHub Actions cache
 
-  integration-test:
+  gh-hosted-integration-test:
     strategy:
       fail-fast: false
-      max-parallel: 6
       matrix:
-        tox-environments:
+        tox-environment:
           - backup-integration
           - charm-integration
           - database-relation-integration
@@ -73,10 +66,17 @@ jobs:
           - tls-integration
           - upgrade-integration
           - upgrade-from-stable-integration
-        agent-versions:
-          - "2.9.45"  # renovate: latest juju 2
-          - "3.1.6"  # renovate: latest juju 3
-    name: ${{ matrix.tox-environments }} | ${{ matrix.agent-versions }}
+        juju-snap-channel:  ["2.9/stable", "3.1/stable"]
+        include:
+          - juju-snap-channel: "3.1/stable"
+            agent-version: "3.1.6"
+            libjuju-version:  "3.2.2"
+            exclude-mark:  "juju2"
+          - juju-snap-channel: "2.9/stable"
+            agent-version: "2.9.45"
+            libjuju-version:  "2.9.45.0"
+            exclude-mark:  "juju3"
+    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.tox-environment }}
     needs:
       - lint
       - unit-test
@@ -85,41 +85,43 @@ jobs:
     timeout-minutes: 120
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
-      - name: Set channel
+        uses: actions/checkout@v4
+      - name: Install tox & poetry
         run: |
-          juju_channel=$(echo "${{ matrix.agent-versions }}" | cut -c 1-3)
-          echo "channel=${juju_channel}/stable" >> "$GITHUB_ENV"
-          juju_major=$(echo "${{ matrix.agent-versions }}" | cut -c 1)
-          echo "libjuju=juju${juju_major}" >> "$GITHUB_ENV"
+          pipx install tox
+          pipx install poetry
       - name: Setup operator environment
-        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           channel: "1.28-strict/stable"
-          bootstrap-options: "--agent-version ${{ matrix.agent-versions }}"
-          juju-channel: "${{ env.channel }}"
+          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
+          juju-channel: ${{ matrix.juju-snap-channel }}
+      - name: Update python-libjuju version
+        if: ${{ matrix.juju-snap-channel == '2.9/stable' }}
+        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
         with:
           name: ${{ needs.build.outputs.artifact-name }}
-      - name: Select tests
-        id: select-tests
+      - name: Select test stability level
+        id: select-test-stability
         run: |
-          if [ "${{ github.event_name }}" == "schedule" ]
+          if [[ "${{ github.event_name }}" == "schedule" ]]
           then
             echo Running unstable and stable tests
-            echo "mark_expression=" >> $GITHUB_OUTPUT
+            echo "mark_expression=" >> "$GITHUB_OUTPUT"
           else
             echo Skipping unstable tests
-            echo "mark_expression=and not unstable" >> $GITHUB_OUTPUT
+            echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT"
           fi
       - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environments }}-${{ env.libjuju }} -- -m 'not not${{ env.libjuju }} ${{ steps.select-tests.outputs.mark_expression }}' --keep-models
+        run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models
         env:
-          AWS_ACCESS_KEY: "${{ secrets.AWS_ACCESS_KEY }}"
-          AWS_SECRET_KEY: "${{ secrets.AWS_SECRET_KEY }}"
-          GCP_ACCESS_KEY: "${{ secrets.GCP_ACCESS_KEY }}"
-          GCP_SECRET_KEY: "${{ secrets.GCP_SECRET_KEY }}"
-          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
+          SECRETS_FROM_GITHUB: |
+            {
+              "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
+              "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
+              "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
+              "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
+            }

.github/workflows/lib-check.yaml

@@ -14,10 +14,6 @@ on:
       - 'LICENSE'
       - '**.md'
       - 'renovate.json'
-  schedule:
-    - cron: '53 0 * * *' # Daily at 00:53 UTC
-  # Triggered on push to branch "main" by .github/workflows/release.yaml
-  workflow_call:
 
 jobs:
   lib-check:
@@ -27,7 +23,7 @@ jobs:
     if: ${{ github.event.pull_request.head.repo.full_name == 'canonical/postgresql-k8s-operator' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Check libs

.github/workflows/release.yaml

@@ -12,7 +12,6 @@ on:
       - renovate.json
       - poetry.lock
       - pyproject.toml
-      - '.github/**.md'
       - '.github/workflows/ci.yaml'
       - '.github/workflows/lib-check.yaml'
 
@@ -21,6 +20,8 @@ jobs:
     name: Tests
     uses: ./.github/workflows/ci.yaml
     secrets: inherit
+    permissions:
+      actions: write  # Needed to manage GitHub Actions cache
 
   release-libraries:
     name: Release libraries
@@ -41,14 +42,14 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v5.1.2
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v5.1.2
     with:
       channel: 14/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@f398b1deef66493fc2018625bcbfa0078b8627bc # v5
+    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v5.1.2
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

.gitignore

@@ -5,6 +5,8 @@ build/
 .coverage
 __pycache__/
 *.py[cod]
+requirements.txt
+requirements-last-build.txt
 
 # PyCharm project folder.
 .idea/

charmcraft.yaml

@@ -8,6 +8,13 @@ bases:
       channel: "22.04"
 parts:
   charm:
+    override-pull: |
+      craftctl default
+      if [[ ! -f requirements.txt ]]
+      then
+          echo 'ERROR: Use "tox run -e build" instead of calling "charmcraft pack" directly' >&2
+          exit 1
+      fi
     build-packages:
       - libffi-dev
       - libssl-dev