backport: KF-6217-use-cache-builds-dev-branch into main (#163)

* build: refactor charmcraft.yaml to use st124 notation (#158)

* build: refactor charmcraft.yaml to use st124 notation

In order to be able to use the data-platform-workflows, specially to be
compatible with charmcraft cache, the charmcraft.yaml files of all charms
in this repository need to follow st124 notation.

Fixes #154

* ci, refactor: use data-platform-workflows in on_pull/push workflows (#159)

* ci, refactor: use data-platform-workflows in on_pull/push workflows

This commit enables the usage of build_charm.yaml workflow, which allows independent and cached charm
builds, making the CI more efficient and fast.
This commit also refactors the tests and the integrate workflow to be able to use the charm artefacts
generated by the build_charm.yaml workflow.
In summary this change:
1. Uses build_charm.yaml in on_{pull&push}.yaml to enable individual cached builds
2. Refactors individual and repository level integration tests to be able to use charm artefacts besides building them
3. Modifies integrate.yaml to download the charm artefacts and uses them in the tox integrate command

Fixes #155

* ci: replaces publish.yaml workflow with promote.yaml workflow (#161)

* ci: replaces publish.yaml workflow with promote.yaml workflow

The promote.yaml workflow replaces the old release workflow dispatch (previously known as
release.yaml).
This commit removes the publish.yaml workflow as the release process
is all handled by a recently refactored release.yaml, and promoting
charms is done in a simple workflow dispatch coming from promote.yaml.

Fixes #157

.github/workflows/integrate.yaml → .github/workflows/ci.yaml

@@ -1,24 +1,55 @@
-# reusable workflow triggered by other actions
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
+  pull_request:
+  schedule:
+    - cron: '0 8 * * TUE'
+  # Triggered on push by .github/workflows/release.yaml
   workflow_call:
-    secrets:
-      CHARMCRAFT_CREDENTIALS:
-        required: true
+    outputs:
+      artifact-prefix:
+        description: build_charm.yaml `artifact-prefix` output
+        value: ${{ jobs.build.outputs.artifact-prefix }}
+      charm-paths:
+        description: paths for all charms in this repo
+        value: ${{ jobs.get-charm-paths-channel.outputs.charm-paths }}
+      channel:
+        description: Charmhub channel the charms are released to
+        value: ${{ jobs.get-charm-paths-channel.outputs.charm-channel }}
 
 jobs:
+  get-charm-paths-channel:
+    name: Get charm paths and charmhub channel
+    runs-on: ubuntu-latest
+    outputs:
+      charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }}
+      charm-channel: ${{ steps.select-channel.outputs.name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get paths for all charms in this repo
+        id: get-charm-paths
+        uses: canonical/kubeflow-ci/actions/get-charm-paths@main
+      - name: Select charmhub channel
+        uses: canonical/charming-actions/channel@2.6.2
+        id: select-channel
+
   lib-check:
     name: Check libraries
+    needs:
+      - get-charm-paths-channel
     strategy:
       matrix:
-        charm:
-        - tensorboard-controller
-        - tensorboards-web-app
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
     uses: canonical/charmed-kubeflow-workflows/.github/workflows/_quality-checks.yaml@main
     secrets: inherit
     with:
-        charm-path: ./charms/${{ matrix.charm }}
+        charm-path: ${{ matrix.charm }}
 
   lint:
     name: Lint Check
@@ -56,22 +87,34 @@ jobs:
 
   terraform-checks:
     name: Terraform
+    needs:
+      - get-charm-paths-channel
     uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main
     strategy:
       matrix:
-        charm:
-          - tensorboard-controller
-          - tensorboards-web-app
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
     with:
-      charm-path: ./charms/${{ matrix.charm }}
+      charm-path: ${{ matrix.charm }}
       # Skipping the Terraform apply check as the tensorboard-controller goes to Waiting status
       # instead of the expected Blocked or Active. This is currently a limitation of the
       # Terraform re-usable workflows in canonical/charmed-kubeflow-workflows
       # See https://github.com/canonical/charmed-kubeflow-workflows/issues/65
       apply: false
+
+  build:
+    strategy:
+      matrix:
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
+    name: Build charm | ${{ matrix.charm }}
+    needs:
+      - get-charm-paths-channel
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.0
+    with:
+      path-to-charm-directory: ${{ matrix.charm }}
 
   integration:
-    name: Charm-specific Integration tests (microk8s)
+    name: Integration tests (microk8s)
+    needs: build
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -80,18 +123,26 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           channel: 1.31-strict/stable
           juju-channel: 3.6/stable
-          charmcraft-channel: 3.x/stable
           microk8s-addons: dns hostpath-storage ingress metallb:10.64.140.43-10.64.140.49
 
+
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ${{ needs.build.outputs.artifact-prefix }}-*
+          merge-multiple: true
+
       - run: |
-          sg snap_microk8s -c "tox -vve ${{ matrix.charm }}-integration"
+          sg snap_microk8s -c "tox -vve ${{ matrix.charm }}-integration -- --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu@20.04-amd64.charm"
 
       # Collect debug logs if failed
       - name: Dump Juju/K8s logs on failure
@@ -103,6 +154,7 @@ jobs:
 
   deploy:
     name: Integration Test
+    needs: build
     runs-on: ubuntu-20.04
     steps:
     - name: Check out code
@@ -112,13 +164,20 @@ jobs:
       with:
         provider: microk8s
         channel: 1.31-strict/stable
-        charmcraft-channel: 3.x/stable
         juju-channel: 3.6/stable
         microk8s-addons: dns hostpath-storage ingress metallb:10.64.140.43-10.64.140.49
 
+    - name: Download packed charm(s)
+      id: download-charms
+      timeout-minutes: 5
+      uses: actions/download-artifact@v4
+      with:
+        pattern: ${{ needs.build.outputs.artifact-prefix }}-*
+        merge-multiple: true
+
     - name: Run test
       run: |
-        tox -e integration
+        tox -e integration -- --charms-path=${{ github.workspace }}/charms/
                            
       # On failure, capture debugging resources
     - name: Get all
@@ -148,3 +207,21 @@ jobs:
     - name: Get tensorboards-web-app operator logs
       run: kubectl logs -n testing --tail 1000 -ljuju-operator=tensorboards-web-app
       if: failure()
+
+  release:
+    strategy:
+      matrix:
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
+    name: Release charm to Charmhub branch | ${{ matrix.charm }}
+    if: ${{ github.event_name == 'pull_request' }}
+    needs:
+      - get-charm-paths-channel
+      - build
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v29.0.0
+    with:
+      channel: ${{ needs.get-charm-paths-channel.outputs.charm-channel }}
+      artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
+      path-to-charm-directory: ${{ matrix.charm }}
+      create-git-tags: false
+    secrets:
+      charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }}

.github/workflows/promote.yaml

@@ -0,0 +1,31 @@
+# reusable workflow triggered manually
+name: Promote charm to other tracks and channels
+
+on:
+  workflow_dispatch:
+    inputs:
+      destination-channel:
+        description: 'Destination Channel'
+        required: true
+      origin-channel:
+        description: 'Origin Channel'
+        required: true
+      charm-name:
+        description: 'Charm subdirectory name'
+        required: true
+
+jobs:
+  promote-charm:
+    name: Promote charm
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Release charm to channel
+        uses: canonical/charming-actions/release-charm@2.6.2
+        with:
+          credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          destination-channel: ${{ github.event.inputs.destination-channel }}
+          origin-channel: ${{ github.event.inputs.origin-channel }}
+          tag-prefix: ${{ github.event.inputs.charm-name }}
+          charm-path: charms/${{ github.event.inputs.charm-name}}