Merge branch 'main' into cost-1665-node-in-storage-report

.github/dependabot.yaml

@@ -0,0 +1,36 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    target-branch: "main"
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.)
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      ci-dependencies:
+        patterns:
+          - "*" # update all github-actions in single PR
+    ignore:
+      # v4 codecov is broken for dependabot updates because dependabot does not have access to secrets.
+      - dependency-name: "codecov/codecov-action"
+
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      k8s.io-dependencies:
+        patterns:
+          - "*k8s.io*"

.github/workflows/build-and-publish.yaml

@@ -0,0 +1,54 @@
+name: Build and Publish Images
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:  # run on any tag that starts with `v` but not tags that end with `-downstream`
+      - 'v**'
+      - '!v**-downstream'
+
+jobs:
+  build-and-publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+    steps:
+      # Get the repository's code
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            quay.io/project-koku/koku-metrics-operator
+          tags: |
+            type=semver,pattern={{raw}}
+            type=sha,prefix=,enable=${{ github.ref_type == 'tag' }}
+            # set latest tag for default branch
+            type=raw,value=latest
+            type=ref,event=branch
+
+      - name: Login to Quay.io
+        uses: redhat-actions/podman-login@v1
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USER }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+          provenance: false
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/ci-manual.yaml

@@ -0,0 +1,145 @@
+name: CI - user triggered jobs
+
+on:
+  workflow_dispatch:
+    inputs:
+      ibmcloudRegion:
+        description: 'ibm cloud instance region'
+        required: true
+        default: 'us-south'
+        type: choice
+        options:
+        - us-south
+        - ca-tor
+        - us-east
+        - eu-es
+        - eu-gb
+        - jp-tok
+        - br-sao
+  # pull_request_target:
+
+jobs:
+
+  unit-tests-s390x:
+    name: unit-test-zvsi (s390x)
+    runs-on: ubuntu-22.04
+    env:
+      ZVSI_SSH_KEY: koku-ci-ssh-key-${{ github.run_id }}-${{ github.run_number }}
+      ZVSI_VPC_NAME: koku-ci-vpc-${{ github.run_id }}-${{ github.run_number }}
+      ZVSI_PROFILE_NAME: bz2-4x16
+      ZVSI_SUBNET_NAME: koku-ci-subnet-${{ github.run_id }}-${{ github.run_number }}
+      ZVSI_FIP_NAME: koku-ci-fip-${{ github.run_id }}-${{ github.run_number }}
+      ZVSI_INS_NAME: koku-ci-zvsi-${{ github.run_id }}-${{ github.run_number }}
+      ZVSI_IMAGE_NAME: ibm-ubuntu-22-04-3-minimal-s390x-2
+    steps:
+      - name: install ibmcli and setup ibm login
+        run: |
+          curl -fsSL https://clis.cloud.ibm.com/install/linux | sh
+          ibmcloud login -q --apikey ${{ secrets.IBMCLOUD_API_KEY }} -r ${{ inputs.ibmcloudRegion }} | head -3
+          ibmcloud plugin install -f vpc-infrastructure
+      - name: generate ssh key
+        run: |
+          ibmcloud is key-create $ZVSI_SSH_KEY "${{ secrets.ZVSI_PUBLIC_KEY }}" --key-type rsa --resource-group-id ${{ secrets.IBMCLOUD_RESOURCE_GROUP_ID }} -q --output JSON | jq 'del(.public_key,.resource_group)'
+          sleep 2
+      - name: create a virtual private cloud instance
+        run: |
+           ibmcloud is vpc-create $ZVSI_VPC_NAME --resource-group-id ${{ secrets.IBMCLOUD_RESOURCE_GROUP_ID }}  -q --output JSON | jq 'del(.resource_group)'
+           sleep 2
+      - name: select a zone randomly
+        run: |
+           query=.cse_source_ips[$(awk "BEGIN{srand(); print int(rand()*(2-0+1))+0}")].zone.name
+           echo "ZVSI_ZONE=$(ibmcloud is vpc $ZVSI_VPC_NAME --output JSON | jq -r $query)" >> $GITHUB_ENV
+      - name: create a subnet for vpc
+        run: |
+           ibmcloud is subnet-create $ZVSI_SUBNET_NAME $ZVSI_VPC_NAME --ipv4-address-count 256 --zone $ZVSI_ZONE --resource-group-id ${{ secrets.IBMCLOUD_RESOURCE_GROUP_ID }} -q --output JSON | jq 'del(.resource_group)'
+           sleep 2
+      - name: create security group rule for ssh access
+        run: |
+           ibmcloud is security-group-rule-add $(ibmcloud is vpc $ZVSI_VPC_NAME --output JSON | jq -r .default_security_group.id) inbound tcp --port-min 22 --port-max 22 -q --output JSON
+           sleep 2
+      - name: create a zvsi instance
+        run: |
+           ibmcloud is instance-create $ZVSI_INS_NAME $ZVSI_VPC_NAME $ZVSI_ZONE $ZVSI_PROFILE_NAME $ZVSI_SUBNET_NAME --keys $ZVSI_SSH_KEY --image $ZVSI_IMAGE_NAME --resource-group-id ${{ secrets.IBMCLOUD_RESOURCE_GROUP_ID }} -q --output JSON | jq 'del(.resource_group)'
+           sleep 30
+      - name: floating ip addess assignment
+        run: |
+           ibmcloud is floating-ip-reserve $ZVSI_FIP_NAME --resource-group-id ${{ secrets.IBMCLOUD_RESOURCE_GROUP_ID }} --vni $(ibmcloud is instance-network-attachment $ZVSI_INS_NAME $(ibmcloud is instance $ZVSI_INS_NAME -q --output JSON | jq -r .primary_network_interface.id) -q --output JSON | jq -r .virtual_network_interface.id) -q --output JSON | jq 'del(.resource_group)'
+      - name: setup floating ip address for ssh connection
+        run: |
+           echo "ZVSI_FIP_ADD=$(ibmcloud is floating-ip $ZVSI_FIP_NAME -q --output JSON | jq -r .address)" >> $GITHUB_ENV
+      - name: Check ZVSI is online
+        run: |
+           zvsi_status=$(ibmcloud is instance $ZVSI_INS_NAME  -q --output JSON | jq -r .status)
+           if [[ -z "$zvsi_status" ]]; then
+              echo "ZVSI is not created";
+              exit 1
+           fi
+           while [[ $zvsi_status == "pending" || $zvsi_status == "starting" || $zvsi_status == "running" ]]
+            do
+              sleep 5
+              zvsi_status=$(ibmcloud is instance $ZVSI_INS_NAME  -q --output JSON | jq -r .status)
+              if [[ $zvsi_status == "failed" ]]; then
+                echo "zvsi failed to start";
+                exit 1;
+              elif [[ $zvsi_status == "running" ]]; then
+                echo "zvsi is running";
+                break;
+              elif [[ -z "$zvsi_status" ]]; then
+                echo "ZVSI is not created";
+                exit 1
+              fi
+            done
+      - name: go unit test in zvsi
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ env.ZVSI_FIP_ADD }}
+          username: root
+          key: ${{ secrets.ZVSI_PRIVATE_KEY }}
+          script: |
+            apt update -y && apt install -y make
+            git clone -b ${{ github.ref_name }} ${{ github.server_url }}/${{ github.repository }} koku-zvsi-clone
+            cd koku-zvsi-clone && snap install go --classic --channel=$(grep -m 1 go go.mod | cut -d\  -f2)/stable && cd ..
+            adduser --disabled-password --gecos "" runner
+            cp -r koku-zvsi-clone /home/runner/koku-zvsi-clone
+            chmod -R 777 /home/runner/koku-zvsi-clone
+            su runner -c "cd /home/runner/koku-zvsi-clone && make test"
+      - name: destroy zvsi instance
+        if: always()
+        run: |
+          ibmcloud is instance-delete  -f -q $ZVSI_INS_NAME
+          sleep 15
+      - name: Check ZVSI is offline
+        if: always()
+        run: |
+           zvsi_status=$(ibmcloud is instance $ZVSI_INS_NAME  -q --output JSON | jq -r .status)
+           if [[ -z "$zvsi_status" ]]; then
+              echo "ZVSI is destroyed";
+              break
+           fi
+           while [[ $zvsi_status == "running" || $zvsi_status == "deleting" ]]
+            do
+              sleep 5
+              zvsi_status=$(ibmcloud is instance $ZVSI_INS_NAME  -q --output JSON | jq -r .status)
+              if [[ -z "$zvsi_status" ]]; then
+              echo "ZVSI is destroyed";
+              break
+              fi
+            done
+      - name: release floating ip address
+        if: always()
+        run: |
+           ibmcloud is floating-ip-release -f -q $ZVSI_FIP_NAME
+      - name: cleanup subnet
+        if: always()
+        run: |
+           ibmcloud is subnet-delete -f -q $ZVSI_SUBNET_NAME
+           sleep 2
+      - name: cleanup virtual private cloud instance
+        if: always()
+        run: |
+           ibmcloud is vpc-delete -f -q $ZVSI_VPC_NAME
+           sleep 2
+      - name: cleanup ssh key
+        if: always()
+        run: |
+          ibmcloud is key-delete -f -q $ZVSI_SSH_KEY

.github/workflows/ci.yaml

@@ -10,104 +10,87 @@ on:
 
 jobs:
 
-  golangci-lint:
-    name: golangci-lint
+  validate-pr:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.18
+          go-version-file: go.mod
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
+      - name: Verify generated Manifests are up to date
+        run: make verify-manifests
 
   unit-tests:
-    name: Unit Tests
     runs-on: ubuntu-22.04
-
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
       - name: Display build environment
         run: printenv
-
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.18 # The Go version to download (if necessary) and use.
-
-      - name: Install kubebuilder
-        run : |
-          os=$(go env GOOS)
-          arch=$(go env GOARCH)
-          curl -L "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_${os}_${arch}.tar.gz" | tar -xz -C /tmp/
-          sudo mv /tmp/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder
-          export PATH=$PATH:/usr/local/kubebuilder/bin
-
+          go-version-file: go.mod
       - name: Run tests
-        run: |
-          make generate manifests
-          go test ./... -v -coverprofile cover.out -covermode=atomic
-
+        run: make test
       - name: Upload test coverage file
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage
           path: cover.out
 
+  unit-tests-qemu:
+    strategy:
+      matrix:
+        # arch: [arm64, s390x, ppc64le]  # kubebuilder is not available for s390x yet
+        arch: [arm64, ppc64le]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Install QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Run tests via qemu/binfmt
+        run: |
+          go env
+          make test-qemu
+        env:
+          GOARCH: ${{ matrix.arch }}
+
   coverage:
-    name: Coverage
     needs: unit-tests
     runs-on: ubuntu-22.04
     steps:
-
       - name: Checkout
         # this checkout is required for the coverage report. If we don't do this, then
         # the uploaded report is invalid and codecov doesn't know how to process it.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
       - name: Download coverage result from unit-tests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage
-
       - name: Upload coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4.2.0
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           file: ./cover.out
           flags: unittests
-          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
-
-  sonarcloud:
-    name: SonarCloud
-    needs: [golangci-lint,unit-tests]
-    runs-on: ubuntu-20.04
-    steps:
-
-      - name: Checkout
-        # this checkout is required for the coverage report. If we don't do this, then
-        # the uploaded report is invalid and codecov doesn't know how to process it.
-        uses: actions/checkout@v3.1.0
-        with:
-          fetch-depth: 0
-
-      - name: Download coverage result from unit-tests
-        uses: actions/download-artifact@v3
-        with:
-          name: coverage
-
-      - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          plugin: pycoverage  # Only run one plugin even though we don't want any to run.