Decouple automated K8s deployments (#1531)

This change allows deploying nativelink in various new configurations, including cache-only and multi-toolchain setups. The new setup is significantly more modular and lets us add and remove toolchains at runtime. Includes a reordering of the deployment logic to ensure that Alerts get deployed before the GitRepository that triggers them. For demo and CI this comes at the cost of an additional dummy GitRepository. This tradeoff seems well worth it as the previous ordering was entirely wrong and unreliable. Fixes the flakiness of the LRE workflow where it often gets stuck in the `Waiting for Tekton pipelines` loop.
TraceMachina · Dec 10, 2024 · a0ca341 · a0ca341
1 parent 8782c0b
commit a0ca341
Show file tree

Hide file tree

Showing 41 changed files with 532 additions and 299 deletions.
diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -96,16 +96,9 @@ jobs:
           nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF
           apiVersion: kustomize.config.k8s.io/v1beta1
           kind: Kustomization
-          components:
-          - kubernetes/components/operator
+          resources:
+          - deploy/kubernetes-example
           patches:
-          - patch: |-
-              - op: replace
-                path: /spec/path
-                value: ./kubernetes/overlays/lre
-            target:
-              kind: Kustomization
-              name: nativelink
           - patch: |-
               - op: replace
                 path: /spec/url
@@ -119,27 +112,6 @@ jobs:
             target:
               kind: GitRepository
               name: nativelink
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#image
-            target:
-              kind: Alert
-              name: nativelink-image-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-init
-            target:
-              kind: Alert
-              name: nativelink-worker-init-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-lre-cc
-            target:
-              kind: Alert
-              name: nativelink-worker-alert
           EOF
           kubectl apply -k . &&
           rm kustomization.yaml'
@@ -151,6 +123,36 @@ jobs:
             --timeout=15m \
             nativelink-tekton-resources"
 
+      - name: Wait for alerts
+        run: >
+          nix develop --impure --command
+          bash -c "flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-core && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-worker-init && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-lre-cc"
+
+      - name: Trigger pipelines
+        run: |
+          nix develop --impure --command bash -c 'cat > dummy-repo.yaml << EOF
+          apiVersion: source.toolkit.fluxcd.io/v1
+          kind: GitRepository
+          metadata:
+            name: dummy-repository
+            namespace: default
+          spec:
+            interval: 2m
+            url: https://github.com/TraceMachina/nativelink
+            ref:
+              branch: main
+          EOF
+          kubectl apply -f dummy-repo.yaml &&
+          rm dummy-repo.yaml'
+
       - name: Wait for Tekton pipelines
         run: >
           nix develop --impure --command bash << 'EOF'
@@ -171,19 +173,19 @@ jobs:
               -l tekton.dev/pipeline=rebuild-nativelink
           EOF
 
-      - name: Wait for Configmaps
+      - name: Wait for NativeLink Kustomization
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-configmaps"
+            nativelink-core"
 
-      - name: Wait for NativeLink Kustomization
+      - name: Wait for Worker Kustomization
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink"
+            nativelink-lre-cc"
 
       - name: Wait for NativeLink
         run: >
@@ -193,7 +195,7 @@ jobs:
       - name: Wait for worker
         run: >
           nix develop --impure --command
-          bash -c "kubectl rollout status deploy/nativelink-worker"
+          bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc"
 
       - name: Get gateway IPs
         id: gateway-ips

diff --git a/deploy/chromium-example/kustomization.yaml b/deploy/chromium-example/kustomization.yaml
@@ -1,21 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
-
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/chromium
-  target:
-    kind: Kustomization
-    name: nativelink
-- patch: |-
-    - op: replace
-      path: /spec/eventMetadata/flakeOutput
-      value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium
-  target:
-    kind: Alert
-    name: nativelink-worker-alert
+resources:
+- ../../kubernetes/resources/flux
+- ../../kubernetes/resources/nativelink-core
+- ../../kubernetes/resources/gateway-routes
+- ../../kubernetes/workers/resources/worker-init
+- ../../kubernetes/workers/resources/siso-chromium
diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml
@@ -1,33 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
+resources:
+- ../kubernetes-example
 
-# Change this value to deploy custom overlays.
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/lre
-  target:
-    kind: Kustomization
-    name: nativelink
-
-# Modify this value to change the URL of the repository with deployment files.
+# Modify this value to change the repository containing the deployment files.
 #
 # This is usually only necessary if you change deployment YAML files or
 # NativeLink config files. If you only intend to change the Rust sources you can
 # leave this as is and need to ensure that the Alerts below are patched to build
 # your local sources.
+patches:
 - patch: |-
     - op: replace
       path: /spec/url
       value: https://github.com/TraceMachina/nativelink
-# Optionally, change the tracked branch.
-#   - op: replace
-#     path: /spec/ref/branch
-#     value: somecustombranch
+    - op: replace
+      path: /spec/ref/branch
+      value: main
   target:
     kind: GitRepository
     name: nativelink
@@ -43,22 +33,22 @@ patches:
 # outputs from a Pull request.
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#image
   target:
-    kind: Alert
-    name: nativelink-image-alert
+    kind: Kustomization
+    name: nativelink-alert-core
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-init
   target:
-    kind: Alert
-    name: nativelink-worker-init-alert
+    kind: Kustomization
+    name: nativelink-alert-worker-init
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-lre-cc
   target:
-    kind: Alert
-    name: nativelink-worker-alert
+    kind: Kustomization
+    name: nativelink-alert-lre-cc
diff --git a/deploy/kubernetes-example/kustomization.yaml b/deploy/kubernetes-example/kustomization.yaml
@@ -1,21 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
-
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/lre
-  target:
-    kind: Kustomization
-    name: nativelink
-- patch: |-
-    - op: replace
-      path: /spec/eventMetadata/flakeOutput
-      value: github:TraceMachina/nativelink#nativelink-worker-lre-cc
-  target:
-    kind: Alert
-    name: nativelink-worker-alert
+resources:
+- ../../kubernetes/resources/flux
+- ../../kubernetes/resources/nativelink-core
+- ../../kubernetes/resources/gateway-routes
+- ../../kubernetes/workers/resources/worker-init
+- ../../kubernetes/workers/resources/lre-cc
diff --git a/deploy/lre-manual/kustomization.yaml b/deploy/lre-manual/kustomization.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../kubernetes/nativelink
+  - ../../kubernetes/workers/lre-cc
+  - ../../kubernetes/resources/gateway-routes
+
+# Note: This deployment is mainly intended for documentation purposes. If you
+#       intend to use it you'll need to provide image tags here.
+#
+#       NativeLink doesn't support a "latest" tag or similar for reproducibility
+#       reasons. For the latest tags see:
+#         https://github.com/orgs/TraceMachina/packages?repo_name=nativelink
+#
+# See the lre overlay for a setup that autodetects these tags and automatically
+# stays in sync with main. See the `../deploy/dev` directory for development
+# setups.
+images:
+  - name: nativelink
+    newName: ghcr.io/TraceMachina/nativelink
+    newTag: TODO
+  - name: nativelink-worker-init
+    newName: ghcr.io/TraceMachina/nativelink-worker-init
+    newTag: TODO
+  - name: nativelink-worker-lre-cc
+    newName: ghcr.io/TraceMachina/nativelink-worker-lre-cc
+    newTag: TODO
diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml
diff --git a/kubernetes/components/alerts/alert.yaml b/kubernetes/components/alerts/alert.yaml
@@ -0,0 +1,17 @@
+---
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Alert
+metadata:
+  name: nativelink-alert
+  namespace: default
+spec:
+  eventSeverity: info
+  eventSources:
+    - kind: GitRepository
+      name: '*'
+      namespace: default
+  providerRef:
+    name: nativelink-webhook
+  eventMetadata:
+    flakeOutput: ${PLACEHOLDER_FLAKE_OUTPUT}
+    configMapName: ${PLACEHOLDER_CONFIG_MAP_NAME}
diff --git a/...ponents/gateway-routes/kustomization.yaml → ...etes/components/alerts/kustomization.yaml b/...ponents/gateway-routes/kustomization.yaml → ...etes/components/alerts/kustomization.yaml
@@ -1,5 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
+
 resources:
-  - routes.yaml
+- alert.yaml
diff --git a/kubernetes/components/kustomization/flux-kustomization.yaml b/kubernetes/components/kustomization/flux-kustomization.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nativelink
+  namespace: default
+spec:
+  interval: 2m
+  path: PLACEHOLDER_NATIVELINK_KUSTOMIZATION
+  prune: true
+  force: true
+  retryInterval: 20s
+  targetNamespace: default
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: nativelink
+    namespace: default
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: nativelink-image-tags
diff --git a/...es/components/operator/kustomization.yaml → ...mponents/kustomization/kustomization.yaml b/...es/components/operator/kustomization.yaml → ...mponents/kustomization/kustomization.yaml
@@ -1,4 +1,6 @@
+---
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
+
 resources:
-- flux-config.yaml
+  - flux-kustomization.yaml