Adding scheduling decision CRD: "why/why not" api (#307)

This contribution adds an api to cortex that describes why a virtual
machine has been placed on a specific compute host. It exposes the raw
pipeline weights with a kubernetes custom resource, and the associated
operator calculates a human-readable description from the provided
values. The resource is also connected to the nova visualizer so that
decisions can be spectated in the browser removing this dependency on
the mqtt broker.

---------

Co-authored-by: Markus Wieland <44964229+SoWieMarkus@users.noreply.github.com>
Co-authored-by: mblos <156897072+mblos@users.noreply.github.com>

Author: 3 people

.github/workflows/push-charts.yaml

@@ -80,3 +80,21 @@ jobs:
             CHART_PACKAGE=$(ls $CHART_DIR/*.tgz)
             helm push $CHART_PACKAGE oci://${{ env.REGISTRY }}/${{ github.repository }}/charts/
           done
+      - name: Get all changed decisions Chart.yaml files
+        id: changed-chart-yaml-files-decisions
+        uses: tj-actions/changed-files@v47
+        with:
+          files: |
+            decisions/dist/chart/Chart.yaml
+      - name: Push decisions charts to registry
+        if: steps.changed-chart-yaml-files-decisions.outputs.all_changed_files != ''
+        shell: bash
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-chart-yaml-files-decisions.outputs.all_changed_files }}
+        run: |
+          for CHART_FILE in ${ALL_CHANGED_FILES}; do
+            CHART_DIR=$(dirname $CHART_FILE)
+            helm package $CHART_DIR --dependency-update --destination $CHART_DIR
+            CHART_PACKAGE=$(ls $CHART_DIR/*.tgz)
+            helm push $CHART_PACKAGE oci://${{ env.REGISTRY }}/${{ github.repository }}/charts/
+          done

.github/workflows/push-images.yaml

@@ -139,3 +139,44 @@ jobs:
           subject-name: ${{ env.REGISTRY }}/${{ github.repository }}-reservations-operator
           subject-digest: ${{ steps.push_cortex_reservations.outputs.digest }}
           push-to-registry: true
+      # Only build and push the decisions operator image if there are changes
+      # in the decisions directory.
+      - name: Get all changed decisions/ files
+        id: changed_decisions_files
+        uses: tj-actions/changed-files@v47
+        with:
+          files: |
+            decisions/**
+      - name: Docker Meta (Cortex Decisions)
+        if: steps.changed_decisions_files.outputs.all_changed_files != ''
+        id: meta_cortex_decisions
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}-decisions-operator
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+            latest
+      - name: Build and Push Cortex Decisions Operator
+        if: steps.changed_decisions_files.outputs.all_changed_files != ''
+        id: push_cortex_decisions
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.kubebuilder
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta_cortex_decisions.outputs.tags }}
+          labels: ${{ steps.meta_cortex_decisions.outputs.labels }}
+          build-args: |
+            GO_MOD_PATH=decisions
+            GIT_TAG=${{ github.ref_name }}
+            GIT_COMMIT=${{ github.sha }}
+      - name: Generate Artifact Attestation for Cortex Decisions
+        if: steps.changed_decisions_files.outputs.all_changed_files != ''
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ github.repository }}-decisions-operator
+          subject-digest: ${{ steps.push_cortex_decisions.outputs.digest }}
+          push-to-registry: true

.github/workflows/test.yaml

@@ -25,6 +25,8 @@ jobs:
           go test -v ./...
           echo "Testing reservations module..."
           cd reservations && go test -v ./...
+          echo "Testing decisions module..."
+          cd ../decisions && go test -v ./...
 
   test-with-docker:
     # We don't need to run this longer test if the previous one already failed.
@@ -61,13 +63,22 @@ jobs:
             -coverprofile=reservations_profile.cov ./internal/...
           go tool cover -func reservations_profile.cov > reservations_func_coverage.txt
           cd ..
+
+          echo "Running tests for decisions module..."
+          cd decisions
+          go test -v \
+            -coverpkg=./internal/... \
+            -coverprofile=decisions_profile.cov ./internal/...
+          go tool cover -func decisions_profile.cov > decisions_func_coverage.txt
+          cd ..
       - name: Upload coverage files
         uses: actions/upload-artifact@v4
         with:
           name: pr-func-coverage
           path: |
             pr_func_coverage.txt
             reservations/reservations_func_coverage.txt
+            decisions/decisions_func_coverage.txt
       # Steps below are only executed if the workflow is triggered by a pull request
       - name: Delete old coverage comments (PR only)
         if: ${{ github.event_name == 'pull_request' }}
@@ -123,6 +134,19 @@ jobs:
               reservationsCoverageReport = 'No coverage data available';
             }
 
+            // Read decisions module coverage report
+            let decisionsCoverageReport = '';
+            let decisionsCoveragePercentage = 'unknown';
+            try {
+              decisionsCoverageReport = fs.readFileSync('decisions/decisions_func_coverage.txt', 'utf8');
+              const decisionsLines = decisionsCoverageReport.trim().split('\n');
+              const decisionsLastLine = decisionsLines[decisionsLines.length - 1];
+              const decisionsCoverageMatch = decisionsLastLine.match(/total:\s+\(statements\)\s+(\d+\.\d+)%/);
+              decisionsCoveragePercentage = decisionsCoverageMatch ? decisionsCoverageMatch[1] : 'unknown';
+            } catch (error) {
+              decisionsCoverageReport = 'No coverage data available';
+            }
+
             let commentBody = '<!-- coverage-comment -->\n';
             commentBody += '## Test Coverage Report\n\n';
 
@@ -144,6 +168,16 @@ jobs:
             commentBody += '```text\n';
             commentBody += reservationsCoverageReport;
             commentBody += '```\n';
+            commentBody += '</details>\n\n';
+
+            // Decisions module coverage
+            commentBody += '<details>\n';
+            commentBody += '<summary>Coverage in decisions module (decisions/internal/): ';
+            commentBody += decisionsCoveragePercentage;
+            commentBody += '%</summary>\n\n';
+            commentBody += '```text\n';
+            commentBody += decisionsCoverageReport;
+            commentBody += '```\n';
             commentBody += '</details>\n';
 
             // Post the comment

.github/workflows/update-appversion.yml

@@ -30,6 +30,13 @@ jobs:
           files: |
             postgres/**
 
+      - name: Get all changed decisions/ files
+        id: changed_decisions_files
+        uses: tj-actions/changed-files@v47
+        with:
+          files: |
+            decisions/**
+
       # Always bumped
       - name: Update appVersion in cortex-core Chart.yaml
         run: |
@@ -69,3 +76,17 @@ jobs:
           git add reservations/dist/chart/Chart.yaml
           git commit -m "Bump cortex-reservations chart appVersions to ${{ steps.vars.outputs.sha }} [skip ci]" || echo "No changes to commit"
           git push origin HEAD:main
+
+      # Only bumped if there are changes in the decisions directory.
+      - name: Update appVersion in cortex-decisions Chart.yaml
+        if: steps.changed_decisions_files.outputs.all_changed_files != ''
+        run: |
+          sed -i 's/^\([ ]*appVersion:[ ]*\).*/\1"${{ steps.vars.outputs.sha }}"/' decisions/dist/chart/Chart.yaml
+      - name: Commit and push changes for cortex-decisions
+        if: steps.changed_decisions_files.outputs.all_changed_files != ''
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add decisions/dist/chart/Chart.yaml
+          git commit -m "Bump cortex-decisions chart appVersions to ${{ steps.vars.outputs.sha }} [skip ci]" || echo "No changes to commit"
+          git push origin HEAD:main

Tiltfile

@@ -37,12 +37,22 @@ def kubebuilder_binary_files(path):
 docker_build('ghcr.io/cobaltcore-dev/cortex-reservations-operator', '.',
     dockerfile='Dockerfile.kubebuilder',
     build_args={'GO_MOD_PATH': 'reservations'},
-    only=kubebuilder_binary_files('reservations') + ['internal/', 'go.mod', 'go.sum'],
+    only=kubebuilder_binary_files('reservations') + ['internal/', 'decisions/', 'go.mod', 'go.sum'],
 )
 local('sh helm/sync.sh reservations/dist/chart')
 k8s_yaml(helm('reservations/dist/chart', name='cortex-reservations', values=[tilt_values]))
 k8s_resource('reservations-controller-manager', labels=['Reservations'])
 
+########### Decisions Operator & CRDs
+docker_build('ghcr.io/cobaltcore-dev/cortex-decisions-operator', '.',
+    dockerfile='Dockerfile.kubebuilder',
+    build_args={'GO_MOD_PATH': 'decisions'},
+    only=kubebuilder_binary_files('decisions') + ['internal/', 'go.mod', 'go.sum'],
+)
+local('sh helm/sync.sh decisions/dist/chart')
+k8s_yaml(helm('decisions/dist/chart', name='cortex-decisions', values=[tilt_values]))
+k8s_resource('decisions-controller-manager', labels=['Decisions'])
+
 ########### Dev Dependencies
 local('sh helm/sync.sh helm/dev/cortex-prometheus-operator')
 k8s_yaml(helm('./helm/dev/cortex-prometheus-operator', name='cortex-prometheus-operator')) # Operator
@@ -82,6 +92,7 @@ k8s_resource('cortex-plutono', port_forwards=[
 docker_build('ghcr.io/cobaltcore-dev/cortex', '.', only=[
     'internal/', 'commands/', 'main.go', 'go.mod', 'go.sum', 'Makefile',
     'reservations/api/', # API module of the reservations operator needed for the scheduler.
+    'decisions/api/', # API module of the decisions operator needed for the scheduler.
 ])
 docker_build('ghcr.io/cobaltcore-dev/cortex-postgres', 'postgres')
 

commands/checks/nova/checks.go

@@ -271,6 +271,7 @@ func randomRequest(dc datacenter, seed int) api.ExternalSchedulerRequest {
 	slog.Info("using flavor extra specs", "extraSpecs", extraSpecs)
 	request := api.ExternalSchedulerRequest{
 		Spec: api.NovaObject[api.NovaSpec]{Data: api.NovaSpec{
+			InstanceUUID:     "cortex-e2e-tests",
 			AvailabilityZone: az,
 			ProjectID:        project.ID,
 			Flavor: api.NovaObject[api.NovaFlavor]{Data: api.NovaFlavor{