From 237febf9b1ec5e45c862b58c0559c015577a0861 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 5 Dec 2024 11:56:05 -0500 Subject: [PATCH 01/29] chore: return types on sdk (#1512) ## Description Create return types on SDK, removed a frivolous log that does not provide any good info to a user. End to End Test: (See [Pepr Excellent Examples](https://github.com/defenseunicorns/pepr-excellent-examples)) ## Related Issue Fixes #1452 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Signed-off-by: Case Wylie --- src/sdk/sdk.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/sdk/sdk.ts b/src/sdk/sdk.ts index 3b07b9f1a..963b644b0 100644 --- a/src/sdk/sdk.ts +++ b/src/sdk/sdk.ts @@ -3,10 +3,8 @@ import { PeprValidateRequest } from "../lib/validate-request"; import { PeprMutateRequest } from "../lib/mutate-request"; -import { V1OwnerReference } from "@kubernetes/client-node"; -import { GenericKind } from "kubernetes-fluent-client"; -import { K8s, kind } from "kubernetes-fluent-client"; -import Log from "../lib/logger"; +import { V1OwnerReference, V1Container } from "@kubernetes/client-node"; +import { GenericKind, K8s, kind } from "kubernetes-fluent-client"; /** * Returns all containers in a pod @@ -17,7 +15,7 @@ import Log from "../lib/logger"; export function containers( request: PeprValidateRequest | PeprMutateRequest, containerType?: "containers" | "initContainers" | "ephemeralContainers", -) { +): V1Container[] { const containers = request.Raw.spec?.containers || []; const initContainers = request.Raw.spec?.initContainers || []; const ephemeralContainers = request.Raw.spec?.ephemeralContainers || []; @@ -44,6 +42,7 @@ export function containers( * @param reportingComponent The component that is reporting the event, for example "uds.dev/operator" * @param reportingInstance The instance of the component that is reporting the event, for example process.env.HOSTNAME */ + export async function writeEvent( cr: GenericKind, event: Partial, @@ -51,9 +50,7 @@ export async function writeEvent( eventReason: string, reportingComponent: string, reportingInstance: string, -) { - Log.debug(cr.metadata, `Writing event: ${event.message}`); - +): Promise { await K8s(kind.CoreEvent).Create({ type: eventType, reason: eventReason, @@ -109,7 +106,7 @@ export function getOwnerRefFrom( * @param name the name of the resource to sanitize * @returns the sanitized resource name */ -export function sanitizeResourceName(name: string) { +export function sanitizeResourceName(name: string): string { return ( name // The name must be lowercase From da68425288fc62e71dbd9fa0d36785eaf4cc4ac3 Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Thu, 5 Dec 2024 12:07:58 -0600 Subject: [PATCH 02/29] chore: store adjudicator code in adjudicators/ (#1517) ## Description This PR moves adjudicator files to the `adjudicators/` directory. ## Related Issue None, this is a one-off PR to relocate files. ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Co-authored-by: Case Wylie --- .../filter/{ => adjudicators}/adjudicators.test.ts | 11 +++-------- src/lib/filter/{ => adjudicators}/adjudicators.ts | 4 ++-- .../filter/adjudicators/bindingAdjudicators.test.ts | 2 +- .../bindingKubernetesObjectAdjudicators.test.ts | 2 +- .../filter/adjudicators/requestAdjudicators.test.ts | 2 +- src/lib/filter/filter.ts | 2 +- src/lib/helpers.ts | 2 +- 7 files changed, 10 insertions(+), 15 deletions(-) rename src/lib/filter/{ => adjudicators}/adjudicators.test.ts (98%) rename src/lib/filter/{ => adjudicators}/adjudicators.ts (98%) diff --git a/src/lib/filter/adjudicators.test.ts b/src/lib/filter/adjudicators/adjudicators.test.ts similarity index 98% rename from src/lib/filter/adjudicators.test.ts rename to src/lib/filter/adjudicators/adjudicators.test.ts index 041d90b46..ffb6a918d 100644 --- a/src/lib/filter/adjudicators.test.ts +++ b/src/lib/filter/adjudicators/adjudicators.test.ts @@ -22,14 +22,9 @@ import { uncarryableNamespace, } from "./adjudicators"; import { KubernetesObject } from "kubernetes-fluent-client"; -import { AdmissionRequest, Binding, DeepPartial } from "../types"; -import { Event, Operation } from "../enums"; -import { - defaultAdmissionRequest, - defaultBinding, - defaultFilters, - defaultKubernetesObject, -} from "./adjudicators/defaultTestObjects"; +import { AdmissionRequest, Binding, DeepPartial } from "../../types"; +import { Event, Operation } from "../../enums"; +import { defaultAdmissionRequest, defaultBinding, defaultFilters, defaultKubernetesObject } from "./defaultTestObjects"; describe("mismatchedName", () => { //[ Binding, KubernetesObject, result ] diff --git a/src/lib/filter/adjudicators.ts b/src/lib/filter/adjudicators/adjudicators.ts similarity index 98% rename from src/lib/filter/adjudicators.ts rename to src/lib/filter/adjudicators/adjudicators.ts index 084cc39f2..038d66a3a 100644 --- a/src/lib/filter/adjudicators.ts +++ b/src/lib/filter/adjudicators/adjudicators.ts @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { Event, Operation } from "../enums"; -import { AdmissionRequest, Binding } from "../../lib/types"; +import { Event, Operation } from "../../enums"; +import { AdmissionRequest, Binding } from "../../types"; import { __, allPass, diff --git a/src/lib/filter/adjudicators/bindingAdjudicators.test.ts b/src/lib/filter/adjudicators/bindingAdjudicators.test.ts index 52a9808d6..535cf1e1e 100644 --- a/src/lib/filter/adjudicators/bindingAdjudicators.test.ts +++ b/src/lib/filter/adjudicators/bindingAdjudicators.test.ts @@ -44,7 +44,7 @@ import { misboundDeleteWithDeletionTimestamp, misboundNamespace, missingName, -} from "../adjudicators"; +} from "./adjudicators"; import { defaultBinding, defaultFilters, defaultKubernetesObject } from "./defaultTestObjects"; describe("definesDeletionTimestamp", () => { diff --git a/src/lib/filter/adjudicators/bindingKubernetesObjectAdjudicators.test.ts b/src/lib/filter/adjudicators/bindingKubernetesObjectAdjudicators.test.ts index 7071c8a6b..4aff9d3c8 100644 --- a/src/lib/filter/adjudicators/bindingKubernetesObjectAdjudicators.test.ts +++ b/src/lib/filter/adjudicators/bindingKubernetesObjectAdjudicators.test.ts @@ -13,7 +13,7 @@ import { mismatchedAnnotations, mismatchedLabels, metasMismatch, -} from "../adjudicators"; +} from "./adjudicators"; import { defaultBinding, defaultFilters, defaultKubernetesObject } from "./defaultTestObjects"; describe("mismatchedName", () => { diff --git a/src/lib/filter/adjudicators/requestAdjudicators.test.ts b/src/lib/filter/adjudicators/requestAdjudicators.test.ts index ebb3bff7c..bbc34b08f 100644 --- a/src/lib/filter/adjudicators/requestAdjudicators.test.ts +++ b/src/lib/filter/adjudicators/requestAdjudicators.test.ts @@ -5,7 +5,7 @@ import { expect, describe, it } from "@jest/globals"; import { Operation } from "../../enums"; import { AdmissionRequest } from "../../types"; import { defaultAdmissionRequest } from "./defaultTestObjects"; -import { declaredUid, declaredKind, declaredVersion, declaredGroup, declaredOperation } from "../adjudicators"; +import { declaredUid, declaredKind, declaredVersion, declaredGroup, declaredOperation } from "./adjudicators"; describe("declaredUid", () => { //[ AdmissionRequest, result ] diff --git a/src/lib/filter/filter.ts b/src/lib/filter/filter.ts index cc59c4567..caf7ad11b 100644 --- a/src/lib/filter/filter.ts +++ b/src/lib/filter/filter.ts @@ -38,7 +38,7 @@ import { missingCarriableNamespace, unbindableNamespaces, uncarryableNamespace, -} from "./adjudicators"; +} from "./adjudicators/adjudicators"; /** * shouldSkipRequest determines if a request should be skipped based on the binding filters. diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index 2eb213de5..a30a564f1 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -28,7 +28,7 @@ import { missingCarriableNamespace, unbindableNamespaces, uncarryableNamespace, -} from "./filter/adjudicators"; +} from "./filter/adjudicators/adjudicators"; export function matchesRegex(pattern: string, testString: string): boolean { return new RegExp(pattern).test(testString); From a6360ac678c6374e2a7b9bf38421398b0fd43cee Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 5 Dec 2024 16:12:43 -0500 Subject: [PATCH 03/29] chore: reduce verbosity of logs by eliminating for metric and health (#1519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …heck logs ## Description Anytime I am looking through the Pepr logs to try and find relevant information i find they are polluted with thousands of healthcheck and metric logs. It makes it extremely difficult to find the relevant pieces of information without using specific greps or jq. With that being said, the healthcheck log does not provide any information that is helpful for debugging, neither does the metric log ## Related Issue Fixes #1518 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Signed-off-by: Case Wylie Co-authored-by: Barrett <81570928+btlghrants@users.noreply.github.com> --- src/lib/controller/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/controller/index.ts b/src/lib/controller/index.ts index 2e9e45fab..6838ad44b 100644 --- a/src/lib/controller/index.ts +++ b/src/lib/controller/index.ts @@ -263,6 +263,11 @@ export class Controller { const startTime = Date.now(); res.on("finish", () => { + const excludedRoutes = ["/healthz", "/metrics"]; + if (excludedRoutes.includes(req.originalUrl)) { + return; + } + const elapsedTime = Date.now() - startTime; const message = { uid: req.body?.request?.uid, From 244747d8da7ca37507e1a3000bb6be4d15ac1200 Mon Sep 17 00:00:00 2001 From: Barrett <81570928+btlghrants@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:56:29 -0600 Subject: [PATCH 04/29] test: validate `pepr build` generates a `helm install`-able chart (#1520) ## Description Adds workflow to validate that deploying a Pepr Module via `helm install` works. ## Related Issue Relates to #1514 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- .github/workflows/deploy-helm.yml | 110 ++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 .github/workflows/deploy-helm.yml diff --git a/.github/workflows/deploy-helm.yml b/.github/workflows/deploy-helm.yml new file mode 100644 index 000000000..19c9336fe --- /dev/null +++ b/.github/workflows/deploy-helm.yml @@ -0,0 +1,110 @@ +name: Deploy Test - Helm + +permissions: read-all +on: + workflow_dispatch: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + helm: + name: deploy test + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - name: Set up Kubernetes + uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 + with: + version: 'latest' + + - name: "install k3d" + run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash" + shell: bash + + - name: clone pepr + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: defenseunicorns/pepr + path: pepr + + - name: setup node + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: 22 + cache: "npm" + cache-dependency-path: pepr + + - name: "set env: PEPR" + run: echo "PEPR=${GITHUB_WORKSPACE}/pepr" >> "$GITHUB_ENV" + + - name: install pepr deps + run: | + cd "$PEPR" + npm ci + + - name: build pepr package + image + run: | + cd "$PEPR" + npm run build:image + + - name: "set env: MOD_NAME" + run: | + echo "MOD_NAME=pepr-test-helm" >> "$GITHUB_ENV" + + - name: "set env: MOD_PATH" + run: | + echo "MOD_PATH=${PEPR}/${MOD_NAME}" >> "$GITHUB_ENV" + + - name: init pepr module + run: | + cd "$PEPR" + npx pepr init --name "$MOD_NAME" --description "$MOD_NAME" --skip-post-init --confirm + sed -i 's/uuid": ".*",/uuid": "'$MOD_NAME'",/g' "$MOD_PATH/package.json" + + - name: build pepr module + run: | + cd "$MOD_PATH" + npm install "${PEPR}/pepr-0.0.0-development.tgz" + npx pepr build --custom-image pepr:dev + + - name: "set env: CLUSTER" + run: echo "CLUSTER=$MOD_NAME" >> "$GITHUB_ENV" + + - name: prep test cluster + run: | + k3d cluster create "$CLUSTER" + k3d image import pepr:dev --cluster "$CLUSTER" + + - name: "set env: KUBECONFIG" + run: echo "KUBECONFIG=$(k3d kubeconfig write "$CLUSTER")" >> "$GITHUB_ENV" + + - name: deploy pepr module + run: | + cd "$MOD_PATH" + helm install "$MOD_NAME" "./dist/${MOD_NAME}-chart" --kubeconfig "$KUBECONFIG" + + - name: wait to win + timeout-minutes: 5 + run: | + while : ; do + kubectl get deploy -n pepr-system + ready=$( + kubectl get deploy pepr-${MOD_NAME} -n pepr-system -o jsonpath='{.status.readyReplicas}' + ) + if [ "$ready" = "2" ] ; then break ; fi + sleep 5 + done + while : ; do + kubectl get deploy -n pepr-system + ready=$( + kubectl get deploy pepr-${MOD_NAME}-watcher -n pepr-system -o jsonpath='{.status.readyReplicas}' + ) + if [ "$ready" = "1" ] ; then break ; fi + sleep 5 + done From c9d7227308b83d13df024815d2b3ab3786023fc8 Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Thu, 5 Dec 2024 16:11:04 -0600 Subject: [PATCH 05/29] chore: move `lib/` code related to data collection to `lib/telemetry` (#1522) ## Description This PR moves code related to collecting, transmitting, and analyzing data into a single directory to provide more clarity to the structure of `lib/` contents. ## Related Issue None, this was an emergent PR focused around an organizational change to Pepr. ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- src/lib.ts | 2 +- src/lib/assets/deploy.ts | 2 +- src/lib/assets/destroy.ts | 2 +- src/lib/capability.test.ts | 4 ++-- src/lib/capability.ts | 2 +- src/lib/controller/index.ts | 4 ++-- src/lib/controller/store.ts | 2 +- src/lib/controller/storeCache.ts | 2 +- src/lib/deploymentChecks.ts | 2 +- src/lib/finalizer.ts | 2 +- src/lib/helpers.ts | 2 +- src/lib/mutate-processor.ts | 2 +- src/lib/queue.test.ts | 4 ++-- src/lib/queue.ts | 2 +- src/lib/{ => telemetry}/logger.test.ts | 0 src/lib/{ => telemetry}/logger.ts | 2 +- src/lib/{ => telemetry}/metrics.test.ts | 0 src/lib/{ => telemetry}/metrics.ts | 0 src/lib/utils.ts | 2 +- src/lib/validate-processor.ts | 2 +- src/lib/watch-processor.test.ts | 8 ++++---- src/lib/watch-processor.ts | 4 ++-- src/runtime/controller.ts | 2 +- 23 files changed, 27 insertions(+), 27 deletions(-) rename src/lib/{ => telemetry}/logger.test.ts (100%) rename src/lib/{ => telemetry}/logger.ts (98%) rename src/lib/{ => telemetry}/metrics.test.ts (100%) rename src/lib/{ => telemetry}/metrics.ts (100%) diff --git a/src/lib.ts b/src/lib.ts index 5d3a530b4..429087a5f 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -2,7 +2,7 @@ import { K8s, RegisterKind, kind as a, fetch, fetchStatus, kind } from "kubernet import * as R from "ramda"; import { Capability } from "./lib/capability"; -import Log from "./lib/logger"; +import Log from "./lib/telemetry/logger"; import { PeprModule } from "./lib/module"; import { PeprMutateRequest } from "./lib/mutate-request"; import * as PeprUtils from "./lib/utils"; diff --git a/src/lib/assets/deploy.ts b/src/lib/assets/deploy.ts index 4e21ca2b9..3f933b482 100644 --- a/src/lib/assets/deploy.ts +++ b/src/lib/assets/deploy.ts @@ -7,7 +7,7 @@ import { K8s, kind } from "kubernetes-fluent-client"; import { V1PolicyRule as PolicyRule } from "@kubernetes/client-node"; import { Assets } from "."; -import Log from "../logger"; +import Log from "../telemetry/logger"; import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking"; import { deployment, moduleSecret, namespace, watcher } from "./pods"; import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac"; diff --git a/src/lib/assets/destroy.ts b/src/lib/assets/destroy.ts index 7e3203a31..6d4a80960 100644 --- a/src/lib/assets/destroy.ts +++ b/src/lib/assets/destroy.ts @@ -3,7 +3,7 @@ import { K8s, kind } from "kubernetes-fluent-client"; -import Log from "../logger"; +import Log from "../telemetry/logger"; import { peprStoreCRD } from "./store"; export async function destroyModule(name: string) { diff --git a/src/lib/capability.test.ts b/src/lib/capability.test.ts index 0e49c9eff..478710b31 100644 --- a/src/lib/capability.test.ts +++ b/src/lib/capability.test.ts @@ -1,5 +1,5 @@ import { Capability } from "./capability"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { CapabilityCfg, FinalizeAction, MutateAction, ValidateAction, WatchLogAction } from "./types"; import { a } from "../lib"; import { V1Pod } from "@kubernetes/client-node"; @@ -21,7 +21,7 @@ jest.mock("./module", () => ({ })); // Mock logger globally -jest.mock("./logger", () => ({ +jest.mock("./telemetry/logger", () => ({ __esModule: true, default: { info: jest.fn(), diff --git a/src/lib/capability.ts b/src/lib/capability.ts index c9120053a..8309520f6 100644 --- a/src/lib/capability.ts +++ b/src/lib/capability.ts @@ -3,7 +3,7 @@ import { GenericClass, GroupVersionKind, modelToGroupVersionKind } from "kubernetes-fluent-client"; import { pickBy } from "ramda"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { isBuildMode, isDevMode, isWatchMode } from "./module"; import { PeprStore, Storage } from "./storage"; import { OnSchedule, Schedule } from "./schedule"; diff --git a/src/lib/controller/index.ts b/src/lib/controller/index.ts index 6838ad44b..1c3d72ded 100644 --- a/src/lib/controller/index.ts +++ b/src/lib/controller/index.ts @@ -7,8 +7,8 @@ import https from "https"; import { Capability } from "../capability"; import { MutateResponse, ValidateResponse } from "../k8s"; -import Log from "../logger"; -import { metricsCollector, MetricsCollector } from "../metrics"; +import Log from "../telemetry/logger"; +import { metricsCollector, MetricsCollector } from "../telemetry/metrics"; import { ModuleConfig, isWatchMode } from "../module"; import { mutateProcessor } from "../mutate-processor"; import { validateProcessor } from "../validate-processor"; diff --git a/src/lib/controller/store.ts b/src/lib/controller/store.ts index 5c05ec860..6075b4070 100644 --- a/src/lib/controller/store.ts +++ b/src/lib/controller/store.ts @@ -7,7 +7,7 @@ import { startsWith } from "ramda"; import { Capability } from "../capability"; import { Store } from "../k8s"; -import Log, { redactedPatch, redactedStore } from "../logger"; +import Log, { redactedPatch, redactedStore } from "../telemetry/logger"; import { DataOp, DataSender, DataStore, Storage } from "../storage"; import { fillStoreCache, sendUpdatesAndFlushCache } from "./storeCache"; diff --git a/src/lib/controller/storeCache.ts b/src/lib/controller/storeCache.ts index 3293a6be4..50340f107 100644 --- a/src/lib/controller/storeCache.ts +++ b/src/lib/controller/storeCache.ts @@ -1,5 +1,5 @@ import { DataOp } from "../storage"; -import Log from "../logger"; +import Log from "../telemetry/logger"; import { K8s } from "kubernetes-fluent-client"; import { Store } from "../k8s"; import { StatusCodes } from "http-status-codes"; diff --git a/src/lib/deploymentChecks.ts b/src/lib/deploymentChecks.ts index 282eb5f0d..f945c2cc1 100644 --- a/src/lib/deploymentChecks.ts +++ b/src/lib/deploymentChecks.ts @@ -1,7 +1,7 @@ // check to see if all replicas are ready for all deployments in the pepr-system namespace import { K8s, kind } from "kubernetes-fluent-client"; -import Log from "./logger"; +import Log from "./telemetry/logger"; // returns true if all deployments are ready, false otherwise export async function checkDeploymentStatus(namespace: string) { diff --git a/src/lib/finalizer.ts b/src/lib/finalizer.ts index dbd23e7c6..2e608bd2d 100644 --- a/src/lib/finalizer.ts +++ b/src/lib/finalizer.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2023-Present The Pepr Authors import { K8s, KubernetesObject, RegisterKind } from "kubernetes-fluent-client"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { Binding, DeepPartial } from "./types"; import { Operation } from "./enums"; import { PeprMutateRequest } from "./mutate-request"; diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index a30a564f1..597a5d3de 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2023-Present The Pepr Authors import { KubernetesObject } from "kubernetes-fluent-client"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { Binding, CapabilityExport } from "./types"; import { sanitizeResourceName } from "../sdk/sdk"; import { diff --git a/src/lib/mutate-processor.ts b/src/lib/mutate-processor.ts index 2d4abd348..941a95817 100644 --- a/src/lib/mutate-processor.ts +++ b/src/lib/mutate-processor.ts @@ -9,7 +9,7 @@ import { Errors } from "./errors"; import { shouldSkipRequest } from "./filter/filter"; import { MutateResponse } from "./k8s"; import { AdmissionRequest } from "./types"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { ModuleConfig } from "./module"; import { PeprMutateRequest } from "./mutate-request"; import { base64Encode, convertFromBase64Map, convertToBase64Map } from "./utils"; diff --git a/src/lib/queue.test.ts b/src/lib/queue.test.ts index bc9f93cc0..f4ceeb451 100644 --- a/src/lib/queue.test.ts +++ b/src/lib/queue.test.ts @@ -2,8 +2,8 @@ import { afterEach, describe, expect, jest, it } from "@jest/globals"; import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types"; import { Queue } from "./queue"; -import Log from "./logger"; -jest.mock("./logger"); +import Log from "./telemetry/logger"; +jest.mock("./telemetry/logger"); describe("Queue", () => { afterEach(() => { diff --git a/src/lib/queue.ts b/src/lib/queue.ts index 577c9c77c..3b6f553fb 100644 --- a/src/lib/queue.ts +++ b/src/lib/queue.ts @@ -3,7 +3,7 @@ import { KubernetesObject } from "@kubernetes/client-node"; import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types"; import { randomBytes } from "node:crypto"; -import Log from "./logger"; +import Log from "./telemetry/logger"; type WatchCallback = (obj: KubernetesObject, phase: WatchPhase) => Promise; diff --git a/src/lib/logger.test.ts b/src/lib/telemetry/logger.test.ts similarity index 100% rename from src/lib/logger.test.ts rename to src/lib/telemetry/logger.test.ts diff --git a/src/lib/logger.ts b/src/lib/telemetry/logger.ts similarity index 98% rename from src/lib/logger.ts rename to src/lib/telemetry/logger.ts index bc063f70c..6f92d5dd2 100644 --- a/src/lib/logger.ts +++ b/src/lib/telemetry/logger.ts @@ -3,7 +3,7 @@ import { Operation } from "fast-json-patch"; import { pino, stdTimeFunctions } from "pino"; -import { Store } from "./k8s"; +import { Store } from "../k8s"; const isPrettyLog = process.env.PEPR_PRETTY_LOGS === "true"; const redactedValue = "**redacted**"; diff --git a/src/lib/metrics.test.ts b/src/lib/telemetry/metrics.test.ts similarity index 100% rename from src/lib/metrics.test.ts rename to src/lib/telemetry/metrics.test.ts diff --git a/src/lib/metrics.ts b/src/lib/telemetry/metrics.ts similarity index 100% rename from src/lib/metrics.ts rename to src/lib/telemetry/metrics.ts diff --git a/src/lib/utils.ts b/src/lib/utils.ts index c2d4ea13a..79b86e9d5 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import Log from "./logger"; +import Log from "./telemetry/logger"; /** Test if a string is ascii or not */ export const isAscii = /^[\s\x20-\x7E]*$/; diff --git a/src/lib/validate-processor.ts b/src/lib/validate-processor.ts index 611fa6411..4f566b520 100644 --- a/src/lib/validate-processor.ts +++ b/src/lib/validate-processor.ts @@ -7,7 +7,7 @@ import { Capability } from "./capability"; import { shouldSkipRequest } from "./filter/filter"; import { ValidateResponse } from "./k8s"; import { AdmissionRequest } from "./types"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { convertFromBase64Map } from "./utils"; import { PeprValidateRequest } from "./validate-request"; import { ModuleConfig } from "./module"; diff --git a/src/lib/watch-processor.test.ts b/src/lib/watch-processor.test.ts index f1244c243..13c50cbf9 100644 --- a/src/lib/watch-processor.test.ts +++ b/src/lib/watch-processor.test.ts @@ -6,20 +6,20 @@ import { K8sInit, WatchPhase } from "kubernetes-fluent-client/dist/fluent/types" import { WatchCfg, WatchEvent, Watcher } from "kubernetes-fluent-client/dist/fluent/watch"; import { Capability } from "./capability"; import { setupWatch, logEvent, queueKey, getOrCreateQueue } from "./watch-processor"; -import Log from "./logger"; -import { metricsCollector } from "./metrics"; +import Log from "./telemetry/logger"; +import { metricsCollector } from "./telemetry/metrics"; type onCallback = (eventName: string | symbol, listener: (msg: string) => void) => void; // Mock the dependencies jest.mock("kubernetes-fluent-client"); -jest.mock("./logger", () => ({ +jest.mock("./telemetry/logger", () => ({ debug: jest.fn(), error: jest.fn(), })); -jest.mock("./metrics", () => ({ +jest.mock("./telemetry/metrics", () => ({ metricsCollector: { initCacheMissWindow: jest.fn(), incCacheMiss: jest.fn(), diff --git a/src/lib/watch-processor.ts b/src/lib/watch-processor.ts index 884b9b4fe..4458d92f4 100644 --- a/src/lib/watch-processor.ts +++ b/src/lib/watch-processor.ts @@ -5,11 +5,11 @@ import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types"; import { Capability } from "./capability"; import { filterNoMatchReason } from "./helpers"; import { removeFinalizer } from "./finalizer"; -import Log from "./logger"; +import Log from "./telemetry/logger"; import { Queue } from "./queue"; import { Binding } from "./types"; import { Event } from "./enums"; -import { metricsCollector } from "./metrics"; +import { metricsCollector } from "./telemetry/metrics"; // stores Queue instances const queues: Record> = {}; diff --git a/src/runtime/controller.ts b/src/runtime/controller.ts index 5b4bfa3ae..ae30a8a69 100644 --- a/src/runtime/controller.ts +++ b/src/runtime/controller.ts @@ -8,7 +8,7 @@ import crypto from "crypto"; import fs from "fs"; import { gunzipSync } from "zlib"; import { K8s, kind } from "kubernetes-fluent-client"; -import Log from "../lib/logger"; +import Log from "../lib/telemetry/logger"; import { packageJSON } from "../templates/data.json"; import { peprStoreCRD } from "../lib/assets/store"; import { validateHash } from "../lib/helpers"; From 072eca42f8c73045e010b07c4288b9d0062e0586 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 22:45:04 +0000 Subject: [PATCH 06/29] chore: bump codecov/codecov-action from 5.0.7 to 5.1.1 (#1523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.0.7 to 5.1.1.
Release notes

Sourced from codecov/codecov-action's releases.

v5.1.1

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.1.0...v5.1.1

v5.1.0

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.7...v5.1.0

Changelog

Sourced from codecov/codecov-action's changelog.

v5.1.1

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.1.0..v5.1.1

v5.1.0

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.7..v5.1.0

v5.0.7

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.6..v5.0.7

v5.0.6

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.5..v5.0.6

v5.0.5

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.4..v5.0.5

v5.0.4

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v5.0.3..v5.0.4

v5.0.3

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=codecov/codecov-action&package-manager=github_actions&previous-version=5.0.7&new-version=5.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 641ef774a..62b7a4d30 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -41,7 +41,7 @@ jobs: - run: npm ci - run: npm run test:unit - name: Upload coverage to Codecov - uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7 + uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1 env: CODECOV_TOKEN: ${{ secrets.CODECOV_ORG_TOKEN }} journey: From 2f9b419c453a3b91ddaf44a88a870207151ef8d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 22:45:36 +0000 Subject: [PATCH 07/29] chore: bump trufflesecurity/trufflehog from 3.84.2 to 3.85.0 (#1524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.84.2 to 3.85.0.
Release notes

Sourced from trufflesecurity/trufflehog's releases.

v3.85.0

What's Changed

New Contributors

Full Changelog: https://github.com/trufflesecurity/trufflehog/compare/v3.84.2...v3.85.0

Commits
  • 710d09b updated twilio detector (#3734)
  • 4cd055f Add analysis info for GCP creds (#3727)
  • 00d4619 fix(deps): update module golang.org/x/text to v0.21.0 (#3731)
  • ccac6c2 fix(deps): update module golang.org/x/sync to v0.10.0 (#3730)
  • 2e5a7e6 feat(typeform): add v2 detector for new key formats (#3660)
  • 2169808 chore(deps): update dependency go to v1.23.4 (#3726)
  • f944716 fix(deps): update module github.com/getsentry/sentry-go to v0.30.0 (#3725)
  • 596d5f0 Add additional canary ID (#3720)
  • 42f01f0 fix(deps): update module github.com/aymanbagabas/go-osc52 to v2 (#3715)
  • 22032f7 [refactor] - detectorKeywordMatcher initialization (#3687)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=trufflesecurity/trufflehog&package-manager=github_actions&previous-version=3.84.2&new-version=3.85.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/secret-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 6feccab2d..5bf94cf77 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -23,6 +23,6 @@ jobs: with: fetch-depth: 0 - name: Default Secret Scanning - uses: trufflesecurity/trufflehog@35943b41905eb1195f021955da17c233ed555e24 # main + uses: trufflesecurity/trufflehog@710d09ba85a0b34cea5592f3a42aae7db5d1a279 # main with: extra_args: --debug --no-verification # Warn on potential violations From a6b66201ae023800b509aebf99c3466224645cfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 22:15:52 -0500 Subject: [PATCH 08/29] chore: bump express from 4.21.1 to 4.21.2 in the production-dependencies group (#1525) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the production-dependencies group with 1 update: [express](https://github.com/expressjs/express). Updates `express` from 4.21.1 to 4.21.2
Release notes

Sourced from express's releases.

4.21.2

What's Changed

Full Changelog: https://github.com/expressjs/express/compare/4.21.1...4.21.2

Changelog

Sourced from express's changelog.

4.21.2 / 2024-11-06

  • deps: path-to-regexp@0.1.12
    • Fix backtracking protection
  • deps: path-to-regexp@0.1.11
    • Throws an error on invalid path values
Commits
Maintainer changes

This version was pushed to npm by jonchurch, a new releaser for express since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=express&package-manager=npm_and_yarn&previous-version=4.21.1&new-version=4.21.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 20 ++++++++++++-------- package.json | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 31a98d38c..a7379b312 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@types/ramda": "0.30.2", - "express": "4.21.1", + "express": "4.21.2", "fast-json-patch": "3.1.1", "follow-redirects": "1.15.9", "http-status-codes": "^2.3.0", @@ -4828,9 +4828,9 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "license": "MIT", "dependencies": { "accepts": "~1.3.8", @@ -4852,7 +4852,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -4867,6 +4867,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/express/node_modules/debug": { @@ -8347,9 +8351,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, "node_modules/path-type": { diff --git a/package.json b/package.json index f329b9f18..e5c844e42 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ }, "dependencies": { "@types/ramda": "0.30.2", - "express": "4.21.1", + "express": "4.21.2", "fast-json-patch": "3.1.1", "follow-redirects": "1.15.9", "http-status-codes": "^2.3.0", From 921b0b4d5645f97157d7d575ded8f97fadbb17d9 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 6 Dec 2024 11:51:23 -0500 Subject: [PATCH 09/29] chore: return types on src/lib/assets/index.ts src/lib/controller/index.ts src/lib/mutate-request.ts (#1515) ## Description Stricter typing to catch type errors in src/lib/assets/index.ts src/lib/controller/index.ts src/lib/mutate-request.ts ## Related Issue Fixes #1513 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --------- Signed-off-by: Case Wylie Co-authored-by: Sam Mayer --- src/lib/assets/index.ts | 38 ++++++++++++++++++------------------- src/lib/controller/index.ts | 16 +++++++++------- src/lib/mutate-request.ts | 22 ++++++++++----------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/lib/assets/index.ts b/src/lib/assets/index.ts index 864ac2f1d..78d0947d8 100644 --- a/src/lib/assets/index.ts +++ b/src/lib/assets/index.ts @@ -51,7 +51,7 @@ function createWebhookYaml( ); } -function helmLayout(basePath: string, unique: string) { +function helmLayout(basePath: string, unique: string): Record> { const helm: Record> = { dirs: { chart: resolve(`${basePath}/${unique}-chart`), @@ -119,20 +119,20 @@ export class Assets { this.apiToken = crypto.randomBytes(32).toString("hex"); } - setHash = (hash: string) => { + setHash = (hash: string): void => { this.hash = hash; }; - deploy = async (force: boolean, webhookTimeout?: number) => { + deploy = async (force: boolean, webhookTimeout?: number): Promise => { this.capabilities = await loadCapabilities(this.path); await deploy(this, force, webhookTimeout); }; - zarfYaml = (path: string) => zarfYaml(this, path); + zarfYaml = (path: string): string => zarfYaml(this, path); - zarfYamlChart = (path: string) => zarfYamlChart(this, path); + zarfYamlChart = (path: string): string => zarfYamlChart(this, path); - allYaml = async (imagePullSecret?: string) => { + allYaml = async (imagePullSecret?: string): Promise => { this.capabilities = await loadCapabilities(this.path); // give error if namespaces are not respected for (const capability of this.capabilities) { @@ -143,7 +143,7 @@ export class Assets { }; /* eslint max-statements: ["warn", 21] */ - generateHelmChart = async (basePath: string) => { + generateHelmChart = async (basePath: string): Promise => { const helm = helmLayout(basePath, this.config.uuid); try { @@ -156,18 +156,18 @@ export class Assets { const code = await fs.readFile(this.path); const pairs: [string, () => string][] = [ - [helm.files.chartYaml, () => dedent(chartYaml(this.config.uuid, this.config.description || ""))], - [helm.files.namespaceYaml, () => dedent(nsTemplate())], - [helm.files.watcherServiceYaml, () => toYaml(watcherService(this.name))], - [helm.files.admissionServiceYaml, () => toYaml(service(this.name))], - [helm.files.tlsSecretYaml, () => toYaml(tlsSecret(this.name, this.tls))], - [helm.files.apiTokenSecretYaml, () => toYaml(apiTokenSecret(this.name, this.apiToken))], - [helm.files.storeRoleYaml, () => toYaml(storeRole(this.name))], - [helm.files.storeRoleBindingYaml, () => toYaml(storeRoleBinding(this.name))], - [helm.files.clusterRoleYaml, () => dedent(clusterRoleTemplate())], - [helm.files.clusterRoleBindingYaml, () => toYaml(clusterRoleBinding(this.name))], - [helm.files.serviceAccountYaml, () => toYaml(serviceAccount(this.name))], - [helm.files.moduleSecretYaml, () => toYaml(moduleSecret(this.name, code, this.hash))], + [helm.files.chartYaml, (): string => dedent(chartYaml(this.config.uuid, this.config.description || ""))], + [helm.files.namespaceYaml, (): string => dedent(nsTemplate())], + [helm.files.watcherServiceYaml, (): string => toYaml(watcherService(this.name))], + [helm.files.admissionServiceYaml, (): string => toYaml(service(this.name))], + [helm.files.tlsSecretYaml, (): string => toYaml(tlsSecret(this.name, this.tls))], + [helm.files.apiTokenSecretYaml, (): string => toYaml(apiTokenSecret(this.name, this.apiToken))], + [helm.files.storeRoleYaml, (): string => toYaml(storeRole(this.name))], + [helm.files.storeRoleBindingYaml, (): string => toYaml(storeRoleBinding(this.name))], + [helm.files.clusterRoleYaml, (): string => dedent(clusterRoleTemplate())], + [helm.files.clusterRoleBindingYaml, (): string => toYaml(clusterRoleBinding(this.name))], + [helm.files.serviceAccountYaml, (): string => toYaml(serviceAccount(this.name))], + [helm.files.moduleSecretYaml, (): string => toYaml(moduleSecret(this.name, code, this.hash))], ]; await Promise.all(pairs.map(async ([file, content]) => await fs.writeFile(file, content()))); diff --git a/src/lib/controller/index.ts b/src/lib/controller/index.ts index 1c3d72ded..a2dc65563 100644 --- a/src/lib/controller/index.ts +++ b/src/lib/controller/index.ts @@ -78,7 +78,7 @@ export class Controller { } /** Start the webhook server */ - startServer = (port: number) => { + startServer = (port: number): void => { if (this.#running) { throw new Error("Cannot start Pepr module: Pepr module was not instantiated with deferStart=true"); } @@ -133,7 +133,7 @@ export class Controller { }); }; - #bindEndpoints = () => { + #bindEndpoints = (): void => { // Health check endpoint this.#app.get("/healthz", Controller.#healthz); @@ -162,7 +162,7 @@ export class Controller { * @param next The next middleware function * @returns */ - #validateToken = (req: express.Request, res: express.Response, next: NextFunction) => { + #validateToken = (req: express.Request, res: express.Response, next: NextFunction): void => { // Validate the token const { token } = req.params; if (token !== this.#token) { @@ -183,7 +183,7 @@ export class Controller { * @param req the incoming request * @param res the outgoing response */ - #metrics = async (req: express.Request, res: express.Response) => { + #metrics = async (req: express.Request, res: express.Response): Promise => { try { // https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#basic-info res.set("Content-Type", "text/plain; version=0.0.4"); @@ -200,7 +200,9 @@ export class Controller { * @param admissionKind the type of admission request * @returns the request handler */ - #admissionReq = (admissionKind: "Mutate" | "Validate") => { + #admissionReq = ( + admissionKind: "Mutate" | "Validate", + ): ((req: express.Request, res: express.Response) => Promise) => { // Create the admission request handler return async (req: express.Request, res: express.Response) => { // Start the metrics timer @@ -259,7 +261,7 @@ export class Controller { * @param res the outgoing response * @param next the next middleware function */ - static #logger(req: express.Request, res: express.Response, next: express.NextFunction) { + static #logger(req: express.Request, res: express.Response, next: express.NextFunction): void { const startTime = Date.now(); res.on("finish", () => { @@ -288,7 +290,7 @@ export class Controller { * @param req the incoming request * @param res the outgoing response */ - static #healthz(req: express.Request, res: express.Response) { + static #healthz(req: express.Request, res: express.Response): void { try { res.send("OK"); } catch (err) { diff --git a/src/lib/mutate-request.ts b/src/lib/mutate-request.ts index 655cc3bcc..99d36bc1a 100644 --- a/src/lib/mutate-request.ts +++ b/src/lib/mutate-request.ts @@ -11,19 +11,19 @@ export class PeprMutateRequest { Raw: T; #input: AdmissionRequest; - get PermitSideEffects() { + get PermitSideEffects(): boolean { return !this.#input.dryRun; } - get IsDryRun() { + get IsDryRun(): boolean | undefined { return this.#input.dryRun; } - get OldResource() { + get OldResource(): KubernetesObject | undefined { return this.#input.oldObject; } - get Request() { + get Request(): AdmissionRequest { return this.#input; } @@ -42,11 +42,11 @@ export class PeprMutateRequest { } } - Merge = (obj: DeepPartial) => { + Merge = (obj: DeepPartial): void => { this.Raw = mergeDeepRight(this.Raw, obj) as unknown as T; }; - SetLabel = (key: string, value: string) => { + SetLabel = (key: string, value: string): this => { const ref = this.Raw; ref.metadata = ref.metadata ?? {}; ref.metadata.labels = ref.metadata.labels ?? {}; @@ -54,7 +54,7 @@ export class PeprMutateRequest { return this; }; - SetAnnotation = (key: string, value: string) => { + SetAnnotation = (key: string, value: string): this => { const ref = this.Raw; ref.metadata = ref.metadata ?? {}; ref.metadata.annotations = ref.metadata.annotations ?? {}; @@ -62,25 +62,25 @@ export class PeprMutateRequest { return this; }; - RemoveLabel = (key: string) => { + RemoveLabel = (key: string): this => { if (this.Raw.metadata?.labels?.[key]) { delete this.Raw.metadata.labels[key]; } return this; }; - RemoveAnnotation = (key: string) => { + RemoveAnnotation = (key: string): this => { if (this.Raw.metadata?.annotations?.[key]) { delete this.Raw.metadata.annotations[key]; } return this; }; - HasLabel = (key: string) => { + HasLabel = (key: string): boolean => { return this.Raw.metadata?.labels?.[key] !== undefined; }; - HasAnnotation = (key: string) => { + HasAnnotation = (key: string): boolean => { return this.Raw.metadata?.annotations?.[key] !== undefined; }; } From 4738896b1840a163279a5d3039b50c6e55bffe71 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 6 Dec 2024 16:48:26 -0500 Subject: [PATCH 10/29] chore: returns on utils,queue,cosign (#1528) ## Description As we continue to strictly type, this is the next map of files that need return types to improve our typing system ## Related Issue Fixes #1526 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Signed-off-by: Case Wylie --- src/lib/queue.ts | 16 ++++++++++++---- src/lib/utils.ts | 10 +++++----- src/sdk/cosign.ts | 8 ++++---- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/lib/queue.ts b/src/lib/queue.ts index 3b6f553fb..ee53abbec 100644 --- a/src/lib/queue.ts +++ b/src/lib/queue.ts @@ -29,11 +29,19 @@ export class Queue { this.#uid = `${Date.now()}-${randomBytes(2).toString("hex")}`; } - label() { + label(): { name: string; uid: string } { return { name: this.#name, uid: this.#uid }; } - stats() { + stats(): { + queue: { + name: string; + uid: string; + }; + stats: { + length: number; + }; + } { return { queue: this.label(), stats: { @@ -51,7 +59,7 @@ export class Queue { * @param reconcile The callback to enqueue for reconcile * @returns A promise that resolves when the object is reconciled */ - enqueue(item: K, phase: WatchPhase, reconcile: WatchCallback) { + enqueue(item: K, phase: WatchPhase, reconcile: WatchCallback): Promise { const note = { queue: this.label(), item: { @@ -73,7 +81,7 @@ export class Queue { * * @returns A promise that resolves when the webapp is reconciled */ - async #dequeue() { + async #dequeue(): Promise { // If there is a pending promise, do nothing if (this.#pendingPromise) { Log.debug("Pending promise, not dequeuing"); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 79b86e9d5..c3d59ca45 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -4,14 +4,14 @@ import Log from "./telemetry/logger"; /** Test if a string is ascii or not */ -export const isAscii = /^[\s\x20-\x7E]*$/; +export const isAscii: RegExp = /^[\s\x20-\x7E]*$/; /** * Encode all ascii values in a map to base64 * @param obj The object to encode * @param skip A list of keys to skip encoding */ -export function convertToBase64Map(obj: { data?: Record }, skip: string[]) { +export function convertToBase64Map(obj: { data?: Record }, skip: string[]): void { obj.data = obj.data ?? {}; for (const key in obj.data) { const value = obj.data[key]; @@ -25,7 +25,7 @@ export function convertToBase64Map(obj: { data?: Record }, skip: * @param obj The object to decode * @returns A list of keys that were skipped */ -export function convertFromBase64Map(obj: { data?: Record }) { +export function convertFromBase64Map(obj: { data?: Record }): string[] { const skip: string[] = []; obj.data = obj.data ?? {}; @@ -47,11 +47,11 @@ export function convertFromBase64Map(obj: { data?: Record }) { } /** Decode a base64 string */ -export function base64Decode(data: string) { +export function base64Decode(data: string): string { return Buffer.from(data, "base64").toString("utf-8"); } /** Encode a string to base64 */ -export function base64Encode(data: string) { +export function base64Encode(data: string): string { return Buffer.from(data).toString("base64"); } diff --git a/src/sdk/cosign.ts b/src/sdk/cosign.ts index 868614fa7..aeef3c949 100644 --- a/src/sdk/cosign.ts +++ b/src/sdk/cosign.ts @@ -212,15 +212,15 @@ export async function verifyImage( url: `https://${X.iref.host}/v2/${X.iref.name}/manifests/${X.iref.tag}`, }; - const supportsMediaType = async (url: string, mediaType: string) => { + const supportsMediaType = async (url: string, mediaType: string): Promise => { return (await head(url, mediaType, { ca: tlsCrts }))["content-type"] === mediaType; }; - const canOciV1Manifest = async (manifestUrl: string) => { + const canOciV1Manifest = async (manifestUrl: string): Promise => { return supportsMediaType(manifestUrl, MediaTypeOciV1.Manifest); }; - const canDockerV2Manifest = async (manifestUrl: string) => { + const canDockerV2Manifest = async (manifestUrl: string): Promise => { return supportsMediaType(manifestUrl, MediaTypeDockerV2.Manifest); }; @@ -228,7 +228,7 @@ export async function verifyImage( const manifestResp = await canOciV1Manifest(X.manifest.url) ? await get(X.manifest.url, MediaTypeOciV1.Manifest, {ca: tlsCrts}) : await canDockerV2Manifest(X.manifest.url) ? await get(X.manifest.url, MediaTypeDockerV2.Manifest, {ca: tlsCrts}) : - (() => { throw "Can't pull image manifest with supported MediaType." })(); + (():never => { throw "Can't pull image manifest with supported MediaType." })(); X.manifest.content = manifestResp.body; X.manifest.digest = `sha256:${crypto From 35bf58904f1b5516f46de6f6e86afbd79968124e Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Fri, 6 Dec 2024 15:50:33 -0600 Subject: [PATCH 11/29] chore(testing): verify pepr can be deployed with zarf (#1531) ## Description This PR adds a test to ensure pepr can be deployed with zarf. ## Related Issue Fixes #1516 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- .github/workflows/deploy-zarf.yml | 110 ++++++++++++++++++ .../scripts/check-deployment-readiness.sh | 48 ++++++++ .gitignore | 2 +- 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy-zarf.yml create mode 100755 .github/workflows/scripts/check-deployment-readiness.sh diff --git a/.github/workflows/deploy-zarf.yml b/.github/workflows/deploy-zarf.yml new file mode 100644 index 000000000..248f904d4 --- /dev/null +++ b/.github/workflows/deploy-zarf.yml @@ -0,0 +1,110 @@ +name: Deploy Test - Zarf + +permissions: read-all +on: + workflow_dispatch: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +env: + MOD_NAME: pepr-test-zarf + +jobs: + zarf: + name: deploy test + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - name: Set up Kubernetes + uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 + with: + version: 'latest' + + - name: "install k3d" + run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash" + shell: bash + + - name: Install The Latest Release Version of Zarf + uses: defenseunicorns/setup-zarf@10e539efed02f75ec39eb8823e22a5c795f492ae #v1.0.1 + with: + download-init-package: true + + - name: clone pepr + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: defenseunicorns/pepr + path: pepr + + - name: setup node + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: 22 + cache: "npm" + cache-dependency-path: pepr + + - name: "set env: PEPR" + run: echo "PEPR=${GITHUB_WORKSPACE}/pepr" >> "$GITHUB_ENV" + + - name: Install Pepr Dependencies + run: | + cd "$PEPR" + npm ci + + - name: Build Pepr Package + Image + run: | + cd "$PEPR" + npm run build:image + + - name: "set env: MOD_PATH" + run: | + echo "MOD_PATH=${PEPR}/${MOD_NAME}" >> "$GITHUB_ENV" + + - name: Init Pepr Module + run: | + cd "$PEPR" + npx pepr init --name "$MOD_NAME" --description "$MOD_NAME" --skip-post-init --confirm + sed -i 's/uuid": ".*",/uuid": "'$MOD_NAME'",/g' "$MOD_PATH/package.json" + + - name: Build Pepr Module + run: | + cd "$MOD_PATH" + npm install "${PEPR}/pepr-0.0.0-development.tgz" + npx pepr build --custom-image pepr:dev + + - name: "set env: CLUSTER" + run: echo "CLUSTER=$MOD_NAME" >> "$GITHUB_ENV" + + - name: Prepare Test Cluster + run: | + k3d cluster create "$CLUSTER" + k3d image import pepr:dev --cluster "$CLUSTER" + + - name: "set env: KUBECONFIG" + run: echo "KUBECONFIG=$(k3d kubeconfig write "$CLUSTER")" >> "$GITHUB_ENV" + + + - name: Initialize Zarf + run: | + cd "$MOD_PATH" + zarf init --confirm + + - name: Package Pepr Module with Zarf + run: | + cd "$MOD_PATH" + zarf package create --confirm "dist/" + + - name: Deploy Pepr Module with Zarf + run: | + cd "$MOD_PATH" + zarf package deploy --confirm zarf-package-pepr-pepr-test-zarf-amd64-0.0.1.tar.zst + + - name: Check Deployment Readiness + timeout-minutes: 5 + run: | + ${PEPR}/.github/workflows/scripts/check-deployment-readiness.sh pepr-$MOD_NAME diff --git a/.github/workflows/scripts/check-deployment-readiness.sh b/.github/workflows/scripts/check-deployment-readiness.sh new file mode 100755 index 000000000..b1ff34e06 --- /dev/null +++ b/.github/workflows/scripts/check-deployment-readiness.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -euo pipefail + +check_deployment_readiness() { + local deployment_name=$1 + local namespace=$2 + local expected_ready_replicas=$3 + local timeout=${4:-300} # Timeout in seconds (default: 5 minutes) + local interval=${5:-5} # Interval between checks in seconds + local elapsed=0 + + echo "$(date '+%Y-%m-%d %H:%M:%S') - Checking readiness for deployment '$deployment_name' in namespace '$namespace'..." + echo "$(date '+%Y-%m-%d %H:%M:%S') - Using timeout: ${timeout}s, interval: ${interval}s" + + while [ "$elapsed" -lt "$timeout" ]; do + ready_replicas=$(kubectl get deploy "$deployment_name" -n "$namespace" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + ready_replicas=${ready_replicas:-0} # Default to 0 if null + + if [ "$ready_replicas" == "$expected_ready_replicas" ]; then + echo "$(date '+%Y-%m-%d %H:%M:%S') - Deployment '$deployment_name' is ready with $ready_replicas replicas." + return 0 + fi + + echo "$(date '+%Y-%m-%d %H:%M:%S') - Waiting for deployment '$deployment_name' to be ready. Ready replicas: ${ready_replicas:-0}/${expected_ready_replicas}." + kubectl get deploy -n "$namespace" + sleep "$interval" + elapsed=$((elapsed + interval)) + done + + echo "$(date '+%Y-%m-%d %H:%M:%S') - Timeout reached while waiting for deployment '$deployment_name' to be ready." + return 1 +} + +# Define success criteria +expected_pepr_replicas=2 +expected_watcher_replicas=1 +module_name=${1:-} +namespace=${2:-pepr-system} # Default to 'pepr-system' if null + +if [ -z "$module_name" ]; then + echo "Error: Module name MUST be provided as the first argument." + exit 1 +fi + +check_deployment_readiness "$module_name" "$namespace" $expected_pepr_replicas || exit 1 # Check readiness for the first deployment + +check_deployment_readiness "$module_name-watcher" "$namespace" $expected_watcher_replicas || exit 1 # Check readiness for the watcher deployment diff --git a/.gitignore b/.gitignore index 5a7e61780..24f70088f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ node_modules/ stats.html .vscode insecure-tls* -pepr-test-module +pepr-test-* pepr-upgrade-test *.tar *.tgz From 9d5beb14ffda700c938f5f0e4fe7fae1dfae3101 Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Mon, 9 Dec 2024 08:41:02 -0600 Subject: [PATCH 12/29] chore(ci): use standard check for helm & zarf installs (#1541) ## Description Use the same deployment checks to confirm success with helm & zarf. ## Related Issue Fixes #1540 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- .github/workflows/deploy-helm.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/deploy-helm.yml b/.github/workflows/deploy-helm.yml index 19c9336fe..90afb70fd 100644 --- a/.github/workflows/deploy-helm.yml +++ b/.github/workflows/deploy-helm.yml @@ -89,22 +89,7 @@ jobs: cd "$MOD_PATH" helm install "$MOD_NAME" "./dist/${MOD_NAME}-chart" --kubeconfig "$KUBECONFIG" - - name: wait to win + - name: Check Deployment Readiness timeout-minutes: 5 run: | - while : ; do - kubectl get deploy -n pepr-system - ready=$( - kubectl get deploy pepr-${MOD_NAME} -n pepr-system -o jsonpath='{.status.readyReplicas}' - ) - if [ "$ready" = "2" ] ; then break ; fi - sleep 5 - done - while : ; do - kubectl get deploy -n pepr-system - ready=$( - kubectl get deploy pepr-${MOD_NAME}-watcher -n pepr-system -o jsonpath='{.status.readyReplicas}' - ) - if [ "$ready" = "1" ] ; then break ; fi - sleep 5 - done + ${PEPR}/.github/workflows/scripts/check-deployment-readiness.sh pepr-$MOD_NAME From 35024c41998a90aa27f86e0431920008b746bba0 Mon Sep 17 00:00:00 2001 From: Barrett <81570928+btlghrants@users.noreply.github.com> Date: Mon, 9 Dec 2024 08:41:37 -0600 Subject: [PATCH 13/29] refactor: resolve eslint warnings (max-depth, complexity) - src/lib/validate-processor.ts (#1529) ### Describe what should be investigated or refactored Refactor of `src/lib/validate-processor.ts` to reduce complexity warnings: ``` /pepr/src/lib/validate-processor.ts 15:8 warning Async function 'validateProcessor' has a complexity of 13. Maximum allowed is 10 complexity 62:9 warning Blocks are nested too deeply (4). Maximum allowed is 3 max-depth ``` ### Additional context Fixes #1381 In support of #1248 --- src/lib/validate-processor.test.ts | 103 +++++++++++++++++++++++++++++ src/lib/validate-processor.ts | 86 +++++++++++++----------- 2 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 src/lib/validate-processor.test.ts diff --git a/src/lib/validate-processor.test.ts b/src/lib/validate-processor.test.ts new file mode 100644 index 000000000..751b76962 --- /dev/null +++ b/src/lib/validate-processor.test.ts @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present The Pepr Authors + +import { beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { GroupVersionKind, kind, KubernetesObject } from "kubernetes-fluent-client"; +import { AdmissionRequest, Binding, Filters } from "./types"; +import { Event, Operation } from "./enums"; +import { PeprValidateRequest } from "./validate-request"; +import { clone } from "ramda"; +import * as sut from "./validate-processor"; + +const testFilters: Filters = { + annotations: {}, + deletionTimestamp: false, + labels: {}, + name: "", + namespaces: [], + regexName: "^default$", + regexNamespaces: [] as string[], +}; + +const testGroupVersionKind: GroupVersionKind = { + kind: "some-kind", + group: "some-group", +}; + +const testBinding: Binding = { + event: Event.ANY, + filters: testFilters, + kind: testGroupVersionKind, + model: kind.Pod, + isFinalize: false, + isMutate: false, + isQueue: false, + isValidate: false, + isWatch: false, +}; + +export const testAdmissionRequest: AdmissionRequest = { + uid: "some-uid", + kind: { kind: "a-kind", group: "a-group" }, + resource: { group: "some-group", version: "some-version", resource: "some-resource" }, + operation: Operation.CONNECT, + name: "some-name", + userInfo: {}, + object: {}, +}; + +export const testActionMetadata: Record = {}; + +export const testPeprValidateRequest = (admissionRequest: AdmissionRequest) => + new PeprValidateRequest(admissionRequest); + +describe("processRequest", () => { + let binding: Binding; + let actionMetadata: Record; + let peprValidateRequest: PeprValidateRequest; + + beforeEach(() => { + binding = clone(testBinding); + actionMetadata = clone(testActionMetadata); + peprValidateRequest = testPeprValidateRequest(testAdmissionRequest); + }); + + it("responds on successful validation action", async () => { + const cbResult = { + allowed: true, + statusCode: 200, + statusMessage: "yay", + }; + const callback = jest.fn().mockImplementation(() => cbResult) as Binding["validateCallback"]; + binding = { ...clone(testBinding), validateCallback: callback }; + + const result = await sut.processRequest(binding, actionMetadata, peprValidateRequest); + + expect(result).toEqual({ + uid: peprValidateRequest.Request.uid, + allowed: cbResult.allowed, + status: { + code: cbResult.statusCode, + message: cbResult.statusMessage, + }, + }); + }); + + it("responds on unsuccessful validation action", async () => { + const callback = jest.fn().mockImplementation(() => { + throw "oof"; + }) as Binding["validateCallback"]; + binding = { ...clone(testBinding), validateCallback: callback }; + + const result = await sut.processRequest(binding, actionMetadata, peprValidateRequest); + + expect(result).toEqual({ + uid: peprValidateRequest.Request.uid, + allowed: false, + status: { + code: 500, + message: `Action failed with error: "oof"`, + }, + }); + }); +}); diff --git a/src/lib/validate-processor.ts b/src/lib/validate-processor.ts index 4f566b520..b5b74bce8 100644 --- a/src/lib/validate-processor.ts +++ b/src/lib/validate-processor.ts @@ -1,17 +1,56 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { kind } from "kubernetes-fluent-client"; - +import { kind, KubernetesObject } from "kubernetes-fluent-client"; import { Capability } from "./capability"; import { shouldSkipRequest } from "./filter/filter"; import { ValidateResponse } from "./k8s"; -import { AdmissionRequest } from "./types"; +import { AdmissionRequest, Binding } from "./types"; import Log from "./telemetry/logger"; import { convertFromBase64Map } from "./utils"; import { PeprValidateRequest } from "./validate-request"; import { ModuleConfig } from "./module"; +export async function processRequest( + binding: Binding, + actionMetadata: Record, + peprValidateRequest: PeprValidateRequest, +): Promise { + const label = binding.validateCallback!.name; + Log.info(actionMetadata, `Processing validation action (${label})`); + + const valResp: ValidateResponse = { + uid: peprValidateRequest.Request.uid, + allowed: true, // Assume it's allowed until a validation check fails + }; + + try { + // Run the validation callback, if it fails set allowed to false + const callbackResp = await binding.validateCallback!(peprValidateRequest); + valResp.allowed = callbackResp.allowed; + + // If the validation callback returned a status code or message, set it in the Response + if (callbackResp.statusCode || callbackResp.statusMessage) { + valResp.status = { + code: callbackResp.statusCode || 400, + message: callbackResp.statusMessage || `Validation failed for ${name}`, + }; + } + + Log.info(actionMetadata, `Validation action complete (${label}): ${callbackResp.allowed ? "allowed" : "denied"}`); + return valResp; + } catch (e) { + // If any validation throws an error, note the failure in the Response + Log.error(actionMetadata, `Action failed: ${JSON.stringify(e)}`); + valResp.allowed = false; + valResp.status = { + code: 500, + message: `Action failed with error: ${JSON.stringify(e)}`, + }; + return valResp; + } +} + export async function validateProcessor( config: ModuleConfig, capabilities: Capability[], @@ -32,52 +71,21 @@ export async function validateProcessor( for (const { name, bindings, namespaces } of capabilities) { const actionMetadata = { ...reqMetadata, name }; - for (const action of bindings) { + for (const binding of bindings) { // Skip this action if it's not a validation action - if (!action.validateCallback) { + if (!binding.validateCallback) { continue; } - const localResponse: ValidateResponse = { - uid: req.uid, - allowed: true, // Assume it's allowed until a validation check fails - }; - // Continue to the next action without doing anything if this one should be skipped - const shouldSkip = shouldSkipRequest(action, req, namespaces, config?.alwaysIgnore?.namespaces); + const shouldSkip = shouldSkipRequest(binding, req, namespaces, config?.alwaysIgnore?.namespaces); if (shouldSkip !== "") { Log.debug(shouldSkip); continue; } - const label = action.validateCallback.name; - Log.info(actionMetadata, `Processing validation action (${label})`); - - try { - // Run the validation callback, if it fails set allowed to false - const resp = await action.validateCallback(wrapped); - localResponse.allowed = resp.allowed; - - // If the validation callback returned a status code or message, set it in the Response - if (resp.statusCode || resp.statusMessage) { - localResponse.status = { - code: resp.statusCode || 400, - message: resp.statusMessage || `Validation failed for ${name}`, - }; - } - - Log.info(actionMetadata, `Validation action complete (${label}): ${resp.allowed ? "allowed" : "denied"}`); - } catch (e) { - // If any validation throws an error, note the failure in the Response - Log.error(actionMetadata, `Action failed: ${JSON.stringify(e)}`); - localResponse.allowed = false; - localResponse.status = { - code: 500, - message: `Action failed with error: ${JSON.stringify(e)}`, - }; - return [localResponse]; - } - response.push(localResponse); + const resp = await processRequest(binding, actionMetadata, wrapped); + response.push(resp); } } From 1d2751ad9190d4a8d80f99662186edf3af65c225 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Mon, 9 Dec 2024 14:38:16 -0500 Subject: [PATCH 14/29] chore: complexity of monitor (#1542) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Reduces the complexity of `npx pepr monitor`, adds _some_ unit tests, got as much coverage as I could, other than that it works the same ```bash > npx ts-node src/cli.ts monitor 🔀 MUTATE pepr-demo/pepr-demo (2df33bcb-dcc8-4579-9239-5046f748173b) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "remove", "path": "/metadata/labels/remove-me" } ] 🔀 MUTATE pepr-demo/kube-root-ca.crt (ce9bcbe1-b2aa-4b6d-bc50-0b8c711c512b) 🔀 MUTATE pepr-demo-2/pepr-demo-2 (ad47c30d-b4e3-49fd-ae6c-d931358e3b32) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" } ] 🔀 MUTATE pepr-demo/secret-1 (52f20119-13cd-473b-b28d-46ac61113ca2) [ { "op": "replace", "path": "/data/example", "value": "dW5pY29ybiBtYWdpYyAtIG1vZGlmaWVkIGJ5IFBlcHI=" }, { "op": "add", "path": "/data/magic", "value": "Y2hhbmdlLXdpdGhvdXQtZW5jb2Rpbmc=" }, { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" } ] 🔀 MUTATE pepr-demo-2/pepr-ssa-demo (6cd0af90-d77f-4859-b141-605dc36a3375) 🔀 MUTATE pepr-demo-2/kube-root-ca.crt (e3add4bb-c028-474d-91df-607b92d52971) 🔀 MUTATE pepr-demo/example-1 (0771353d-969f-4868-bdaf-5f2d32b48190) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "add", "path": "/metadata/annotations/pepr.dev", "value": "annotations-work-too" }, { "op": "add", "path": "/metadata/labels", "value": { "pepr": "was-here" } } ] 🔀 MUTATE pepr-demo/example-2 (2ad5718d-3693-412c-91c8-98288c9e5e2d) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "add", "path": "/metadata/annotations/pepr.dev", "value": "annotations-work-too" }, { "op": "add", "path": "/metadata/labels", "value": { "pepr": "was-here" } } ] 🔀 MUTATE pepr-demo/example-evil-cm (13e5b5c2-a411-4f6c-9698-ae3251676aef) 🔀 MUTATE pepr-demo/example-3 (02ec1498-2449-4844-9ed0-7f1b400baef7) [ { "op": "add", "path": "/data/username", "value": "system:admin" }, { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "add", "path": "/metadata/annotations/pepr.dev", "value": "making-waves" } ] 🔀 MUTATE pepr-demo/example-4 (bc436d1c-86b2-408b-9ff4-ffcdf4528f7b) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "add", "path": "/metadata/labels", "value": { "pepr.dev/first": "true", "pepr.dev/second": "true", "pepr.dev/third": "true" } } ] 🔀 MUTATE pepr-demo-2/example-4a (4811a195-73e0-4c2d-a4c6-c0a50bce1af1) [ { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" }, { "op": "add", "path": "/metadata/labels", "value": { "pepr.dev/first": "true", "pepr.dev/second": "true", "pepr.dev/third": "true" } } ] 🔀 MUTATE pepr-demo/example-5 (46d0657c-9a5a-47c3-9c3e-49d6d81cb141) [ { "op": "add", "path": "/data/chuck-says", "value": "No one has ever spoken during review of Chuck Norris' code and lived to tell about it." }, { "op": "add", "path": "/metadata/annotations/3b1b7ed6-88f6-54ec-9ae0-0dcc8a432456.pepr.dev~1upgrade-test", "value": "succeeded" } ] ✅ VALIDATE pepr-demo/kube-root-ca.crt (742c156f-6d1a-459c-9dbe-40defb9a4b18) ✅ VALIDATE pepr-demo-2/pepr-ssa-demo (d3ab073f-2789-4324-abd9-2fc7bb9aa552) ✅ VALIDATE pepr-demo-2/kube-root-ca.crt (6d21adab-cafd-4a85-8c1f-827f5adb1049) ✅ VALIDATE pepr-demo/example-1 (37926781-6c5c-4641-98cf-eb855fc19e9d) ✅ VALIDATE pepr-demo/example-2 (34dd49fb-dbfe-4130-a8ba-45d10cb4223b) ✅ VALIDATE pepr-demo/example-2 (34dd49fb-dbfe-4130-a8ba-45d10cb4223b) ❌ VALIDATE pepr-demo/example-evil-cm (fb63c6ed-822d-418b-acb7-9a43f1ec9b59) No evil CM annotations allowed. ✅ VALIDATE pepr-demo/example-3 (ca76fd37-1548-41a6-968e-463ca759c8e9) ✅ VALIDATE pepr-demo/example-4 (814d9145-f499-441b-9fa7-b36db3069e83) ✅ VALIDATE pepr-demo-2/example-4a (583882a3-fb0b-4185-beca-fa1c8bd8ccc7) ✅ VALIDATE pepr-demo/example-5 (900e65d8-530e-47a1-8f06-79fb448eedde) ``` ## Related Issue Fixes #1537 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --------- Signed-off-by: Case Wylie --- src/cli/monitor.test.ts | 167 ++++++++++++++++++++++++++++++++++++++ src/cli/monitor.ts | 173 +++++++++++++++++++++++++--------------- 2 files changed, 275 insertions(+), 65 deletions(-) create mode 100644 src/cli/monitor.test.ts diff --git a/src/cli/monitor.test.ts b/src/cli/monitor.test.ts new file mode 100644 index 000000000..d16feca5f --- /dev/null +++ b/src/cli/monitor.test.ts @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present The Pepr Authors + +import { expect, describe, it, jest, beforeEach, afterEach } from "@jest/globals"; +import { + getLabelsAndErrorMessage, + getK8sLogFromKubeConfig, + processMutateLog, + processValidateLog, +} from "./monitor"; +import { KubeConfig, Log as K8sLog } from "@kubernetes/client-node"; +import { SpiedFunction } from "jest-mock"; + +const payload = { + level: 30, + time: 1733751945893, + pid: 1, + hostname: "test-host", + uid: "test-uid", + namespace: "test-namespace", + name: "test-name", + res: { + allowed: true, + uid: "test-uid", + patch: btoa(JSON.stringify({ key: "value" })), + patchType: "test-patch-type", + warning: "test-warning", + status: { message: "test-message" }, + }, + msg: "Check response", +}; + +jest.mock("@kubernetes/client-node", () => { + const mockKubeConfig = jest.fn(); + mockKubeConfig.prototype.loadFromDefault = jest.fn(); + const mockK8sLog = jest.fn(); + + return { + KubeConfig: mockKubeConfig, + Log: mockK8sLog, + }; +}); + +describe("getK8sLogFromKubeConfig", () => { + it("should create a K8sLog instance from the default KubeConfig", () => { + const result = getK8sLogFromKubeConfig(); + expect(KubeConfig).toHaveBeenCalledTimes(1); + expect(KubeConfig.prototype.loadFromDefault).toHaveBeenCalledTimes(1); + + const kubeConfigInstance = new KubeConfig(); + expect(K8sLog).toHaveBeenCalledWith(kubeConfigInstance); + + expect(result).toBeInstanceOf(K8sLog); + }); +}); + +describe("getLabelsAndErrorMessage", () => { + it.each([ + [ + undefined, + { + labels: ["pepr.dev/controller", "admission"], + errorMessage: "No pods found with admission labels", + }, + ], + ["test", { labels: ["app", "pepr-test"], errorMessage: "No pods found for module test" }], + ["test2", { labels: ["app", "pepr-test2"], errorMessage: "No pods found for module test2" }], + ["test3", { labels: ["app", "pepr-test3"], errorMessage: "No pods found for module test3" }], + ["test4", { labels: ["app", "pepr-test4"], errorMessage: "No pods found for module test4" }], + ])("should return labels and error message", (uuid, expected) => { + const result = getLabelsAndErrorMessage(uuid); + expect(result).toEqual(expected); + }); +}); + +describe("processMutateLog", () => { + let consoleLogSpy: SpiedFunction<{ + (...data: unknown[]): void; + (message?: unknown, ...optionalParams: unknown[]): void; + }>; + + beforeEach(() => { + consoleLogSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("should log a mutation approval with patch details", () => { + processMutateLog( + { + ...payload, + res: { + ...payload.res, + patch: btoa(JSON.stringify({ key: "value" })), + patchType: "JSONPatch", + }, + }, + "test-name", + "test-uid", + ); + + expect(consoleLogSpy).toHaveBeenCalledWith("\n🔀 MUTATE test-name (test-uid)"); + expect(consoleLogSpy).toHaveBeenCalledWith( + expect.stringContaining(JSON.stringify({ key: "value" }, null, 2)), + ); + }); + + it("should log a mutation denial without patch details", () => { + processMutateLog( + { + ...payload, + res: { + ...payload.res, + allowed: false, + patch: btoa(JSON.stringify("something")), + }, + }, + "test-name", + "test-uid", + ); + + expect(consoleLogSpy).toHaveBeenCalledWith("\n🚫 MUTATE test-name (test-uid)"); + expect(consoleLogSpy).not.toHaveBeenCalledWith(expect.stringContaining("{")); + }); +}); + +describe("processValidateLog", () => { + let consoleLogSpy: SpiedFunction<{ + (...data: unknown[]): void; + (message?: unknown, ...optionalParams: unknown[]): void; + }>; + + beforeEach(() => { + consoleLogSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("should log a successful validation", () => { + processValidateLog(payload, "test-name", "test-uid"); + + expect(consoleLogSpy).toHaveBeenCalledWith("\n✅ VALIDATE test-name (test-uid)"); + expect(consoleLogSpy).not.toHaveBeenCalledWith(expect.stringContaining("❌")); + }); + + it("should log a validation failure with error messages", () => { + processValidateLog( + { + ...payload, + res: { + ...payload.res, + allowed: false, + status: { message: "Failure message 1" }, + }, + }, + "test-name", + "test-uid", + ); + + expect(consoleLogSpy).toHaveBeenCalledWith("\n❌ VALIDATE test-name (test-uid)"); + expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining("Failure message 1")); + }); +}); diff --git a/src/cli/monitor.ts b/src/cli/monitor.ts index 4b83811a9..1fafb30f4 100644 --- a/src/cli/monitor.ts +++ b/src/cli/monitor.ts @@ -1,92 +1,51 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { Log as K8sLog, KubeConfig } from "@kubernetes/client-node"; +import { Log as K8sLog, KubeConfig, KubernetesListObject } from "@kubernetes/client-node"; import { K8s, kind } from "kubernetes-fluent-client"; import stream from "stream"; import { ResponseItem } from "../lib/types"; import { RootCmd } from "./root"; -export default function (program: RootCmd) { +interface LogPayload { + namespace: string; + name: string; + res: { + uid: string; + allowed?: boolean; + patch?: string; + patchType?: string; + warnings?: string; + status?: { + message: string; + }; + }; +} + +export default function (program: RootCmd): void { program .command("monitor [module-uuid]") .description("Monitor a Pepr Module") .action(async uuid => { - let labels: string[]; - let errorMessage: string; - - if (!uuid) { - labels = ["pepr.dev/controller", "admission"]; - errorMessage = `No pods found with admission labels`; - } else { - labels = ["app", `pepr-${uuid}`]; - errorMessage = `No pods found for module ${uuid}`; - } + const { labels, errorMessage } = getLabelsAndErrorMessage(uuid); // Get the logs for the `app=pepr-${module}` or `pepr.dev/controller=admission` pod selector - const pods = await K8s(kind.Pod) + const pods: KubernetesListObject = await K8s(kind.Pod) .InNamespace("pepr-system") .WithLabel(labels[0], labels[1]) .Get(); - const podNames = pods.items.flatMap(pod => pod.metadata!.name) as string[]; + // Pods will ways have a metadata and name fields + const podNames: string[] = pods.items.flatMap(pod => pod.metadata!.name || ""); if (podNames.length < 1) { console.error(errorMessage); process.exit(1); } - const kc = new KubeConfig(); - kc.loadFromDefault(); - - const log = new K8sLog(kc); - - const logStream = new stream.PassThrough(); - logStream.on("data", async chunk => { - const respMsg = `"msg":"Check response"`; - // Split the chunk into lines - const lines = await chunk.toString().split("\n"); - - for (const line of lines) { - // Check for `"msg":"Hello Pepr"` - if (!line.includes(respMsg)) continue; - try { - const payload = JSON.parse(line.trim()); - const isMutate = payload.res.patchType || payload.res.warnings; - - const name = `${payload.namespace}${payload.name}`; - const uid = payload.res.uid; - - if (isMutate) { - const plainPatch = - payload.res?.patch !== undefined && payload.res?.patch !== null - ? atob(payload.res.patch) - : ""; - - const patch = plainPatch !== "" && JSON.stringify(JSON.parse(plainPatch), null, 2); - const patchType = payload.res.patchType || payload.res.warnings || ""; - const allowOrDeny = payload.res.allowed ? "🔀" : "🚫"; - console.log(`\n${allowOrDeny} MUTATE ${name} (${uid})`); - patchType.length > 0 && console.log(`\n\u001b[1;34m${patch}\u001b[0m`); - } else { - const failures = Array.isArray(payload.res) ? payload.res : [payload.res]; - - const filteredFailures = failures - .filter((r: ResponseItem) => !r.allowed) - .map((r: ResponseItem) => r.status.message); - - console.log( - `\n${filteredFailures.length > 0 ? "❌" : "✅"} VALIDATE ${name} (${uid})`, - ); - console.log( - filteredFailures.length > 0 ? `\u001b[1;31m${filteredFailures}\u001b[0m` : "", - ); - } - } catch { - // Do nothing - } - } - }); + const log = getK8sLogFromKubeConfig(); + + const logStream = createLogStream(); for (const podName of podNames) { await log.log("pepr-system", podName, "server", logStream, { @@ -97,3 +56,87 @@ export default function (program: RootCmd) { } }); } + +export function getLabelsAndErrorMessage(uuid?: string): { + labels: string[]; + errorMessage: string; +} { + let labels: string[]; + let errorMessage: string; + + if (!uuid) { + labels = ["pepr.dev/controller", "admission"]; + errorMessage = `No pods found with admission labels`; + } else { + labels = ["app", `pepr-${uuid}`]; + errorMessage = `No pods found for module ${uuid}`; + } + + return { labels, errorMessage }; +} + +export function getK8sLogFromKubeConfig(): K8sLog { + const kc = new KubeConfig(); + kc.loadFromDefault(); + return new K8sLog(kc); +} + +function createLogStream(): stream.PassThrough { + const logStream = new stream.PassThrough(); + + logStream.on("data", async chunk => { + const lines = chunk.toString().split("\n"); + const respMsg = `"msg":"Check response"`; + + for (const line of lines) { + if (!line.includes(respMsg)) continue; + processLogLine(line); + } + }); + + return logStream; +} + +function processLogLine(line: string): void { + try { + const payload: LogPayload = JSON.parse(line.trim()); + const isMutate = payload.res.patchType || payload.res.warnings; + const name = `${payload.namespace}${payload.name}`; + const uid = payload.res.uid; + + if (isMutate) { + processMutateLog(payload, name, uid); + } else { + processValidateLog(payload, name, uid); + } + } catch { + // Do nothing + } +} + +export function processMutateLog(payload: LogPayload, name: string, uid: string): void { + const plainPatch = + payload.res.patch !== undefined && payload.res.patch !== null ? atob(payload.res.patch) : ""; + + const patch = plainPatch !== "" && JSON.stringify(JSON.parse(plainPatch), null, 2); + const patchType = payload.res.patchType || payload.res.warnings || ""; + const allowOrDeny = payload.res.allowed ? "🔀" : "🚫"; + + console.log(`\n${allowOrDeny} MUTATE ${name} (${uid})`); + if (patchType.length > 0) { + console.log(`\n\u001b[1;34m${patch}\u001b[0m`); + } +} + +export function processValidateLog(payload: LogPayload, name: string, uid: string): void { + const failures = Array.isArray(payload.res) ? payload.res : [payload.res]; + + const filteredFailures = failures + .filter((r: ResponseItem) => !r.allowed) + .map((r: ResponseItem) => r.status?.message || ""); + + console.log(`\n${filteredFailures.length > 0 ? "❌" : "✅"} VALIDATE ${name} (${uid})`); + if (filteredFailures.length > 0) { + console.log(`\u001b[1;31m${filteredFailures}\u001b[0m`); + } +} From 26a6218f8a6cd583f7c87074be6622c66b8409b7 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Tue, 10 Dec 2024 16:56:58 -0500 Subject: [PATCH 15/29] chore: return types on store and capability (#1555) ## Description return types on Store and Capability ## Related Issue Fixes #1550 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Signed-off-by: Case Wylie Co-authored-by: Sam Mayer --- src/lib/capability.ts | 31 +++++++++++++++++++++---------- src/lib/controller/store.ts | 14 +++++++------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/lib/capability.ts b/src/lib/capability.ts index 8309520f6..e2cbf3970 100644 --- a/src/lib/capability.ts +++ b/src/lib/capability.ts @@ -71,7 +71,7 @@ export class Capability implements CapabilityExport { } }; - public getScheduleStore() { + public getScheduleStore(): Storage { return this.#scheduleStore; } @@ -111,19 +111,19 @@ export class Capability implements CapabilityExport { onReady: this.#scheduleStore.onReady, }; - get bindings() { + get bindings(): Binding[] { return this.#bindings; } - get name() { + get name(): string { return this.#name; } - get description() { + get description(): string { return this.#description; } - get namespaces() { + get namespaces(): string[] { return this.#namespaces || []; } @@ -207,8 +207,19 @@ export class Capability implements CapabilityExport { const bindings = this.#bindings; const prefix = `${this.#name}: ${model.name}`; const commonChain = { WithLabel, WithAnnotation, WithDeletionTimestamp, Mutate, Validate, Watch, Reconcile, Alias }; - const isNotEmpty = (value: object) => Object.keys(value).length > 0; - const log = (message: string, cbString: string) => { + + type CommonChainType = typeof commonChain; + type ExtendedCommonChainType = CommonChainType & { + Alias: (alias: string) => CommonChainType; + InNamespace: (...namespaces: string[]) => BindingWithName; + InNamespaceRegex: (...namespaces: RegExp[]) => BindingWithName; + WithName: (name: string) => BindingFilter; + WithNameRegex: (regexName: RegExp) => BindingFilter; + WithDeletionTimestamp: () => BindingFilter; + }; + + const isNotEmpty = (value: object): boolean => Object.keys(value).length > 0; + const log = (message: string, cbString: string): void => { const filteredObj = pickBy(isNotEmpty, binding.filters); Log.info(`${message} configured for ${binding.event}`, prefix); @@ -329,7 +340,7 @@ export class Capability implements CapabilityExport { isWatch: true, isFinalize: true, event: Event.UPDATE, - finalizeCallback: async (update: InstanceType, logger = aliasLogger) => { + finalizeCallback: async (update: InstanceType, logger = aliasLogger): Promise => { Log.info(`Executing finalize action with alias: ${binding.alias || "no alias provided"}`); return await finalizeCallback(update, logger); }, @@ -380,13 +391,13 @@ export class Capability implements CapabilityExport { return commonChain; } - function Alias(alias: string) { + function Alias(alias: string): CommonChainType { Log.debug(`Adding prefix alias ${alias}`, prefix); binding.alias = alias; return commonChain; } - function bindEvent(event: Event) { + function bindEvent(event: Event): ExtendedCommonChainType { binding.event = event; return { ...commonChain, diff --git a/src/lib/controller/store.ts b/src/lib/controller/store.ts index 6075b4070..159ff8a7e 100644 --- a/src/lib/controller/store.ts +++ b/src/lib/controller/store.ts @@ -25,7 +25,7 @@ export class StoreController { this.#name = name; - const setStorageInstance = (registrationFunction: () => Storage, name: string) => { + const setStorageInstance = (registrationFunction: () => Storage, name: string): void => { const scheduleStore = registrationFunction(); // Bind the store sender to the capability @@ -61,12 +61,12 @@ export class StoreController { ); } - #setupWatch = () => { + #setupWatch = (): void => { const watcher = K8s(Store, { name: this.#name, namespace }).Watch(this.#receive); watcher.start().catch(e => Log.error(e, "Error starting Pepr store watch")); }; - #migrateAndSetupWatch = async (store: Store) => { + #migrateAndSetupWatch = async (store: Store): Promise => { Log.debug(redactedStore(store), "Pepr Store migration"); const data: DataStore = store.data || {}; let storeCache: Record = {}; @@ -96,11 +96,11 @@ export class StoreController { this.#setupWatch(); }; - #receive = (store: Store) => { + #receive = (store: Store): void => { Log.debug(redactedStore(store), "Pepr Store update"); // Wrap the update in a debounced function - const debounced = () => { + const debounced = (): void => { // Base64 decode the data const data: DataStore = store.data || {}; @@ -137,7 +137,7 @@ export class StoreController { this.#sendDebounce = setTimeout(debounced, this.#onReady ? 0 : debounceBackoff); }; - #send = (capabilityName: string) => { + #send = (capabilityName: string): DataSender => { let storeCache: Record = {}; // Create a sender function for the capability to add/remove data from the store @@ -156,7 +156,7 @@ export class StoreController { return sender; }; - #createStoreResource = async (e: unknown) => { + #createStoreResource = async (e: unknown): Promise => { Log.info(`Pepr store not found, creating...`); Log.debug(e); From b69b58d43f45f8c79dc09cf9579178bd991c98ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 22:27:48 +0000 Subject: [PATCH 16/29] chore: bump trufflesecurity/trufflehog from 3.85.0 to 3.86.0 (#1559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.85.0 to 3.86.0.
Release notes

Sourced from trufflesecurity/trufflehog's releases.

v3.86.0

What's Changed

New Contributors

Full Changelog: https://github.com/trufflesecurity/trufflehog/compare/v3.85.0...v3.86.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=trufflesecurity/trufflehog&package-manager=github_actions&previous-version=3.85.0&new-version=3.86.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/secret-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 5bf94cf77..da2e4771a 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -23,6 +23,6 @@ jobs: with: fetch-depth: 0 - name: Default Secret Scanning - uses: trufflesecurity/trufflehog@710d09ba85a0b34cea5592f3a42aae7db5d1a279 # main + uses: trufflesecurity/trufflehog@f726d02330dbcec836fa17f79fa7711fdb3a5cc8 # main with: extra_args: --debug --no-verification # Warn on potential violations From 94b5833cd8e5427da472763846f4d1290b318744 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 22:27:50 +0000 Subject: [PATCH 17/29] chore: bump github/codeql-action from 3.27.6 to 3.27.7 (#1558) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.6 to 3.27.7.
Release notes

Sourced from github/codeql-action's releases.

v3.27.7

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

Note that the only difference between v2 and v3 of the CodeQL Action is the node version they support, with v3 running on node 20 while we continue to release v2 to support running on node 16. For example 3.22.11 was the first v3 release and is functionally identical to 2.22.11. This approach ensures an easy way to track exactly which features are included in different versions, indicated by the minor and patch version numbers.

3.27.7 - 10 Dec 2024

  • We are rolling out a change in December 2024 that will extract the CodeQL bundle directly to the toolcache to improve performance. #2631
  • Update default CodeQL bundle version to 2.20.0. #2636

See the full CHANGELOG.md for more information.

Changelog

Sourced from github/codeql-action's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

Note that the only difference between v2 and v3 of the CodeQL Action is the node version they support, with v3 running on node 20 while we continue to release v2 to support running on node 16. For example 3.22.11 was the first v3 release and is functionally identical to 2.22.11. This approach ensures an easy way to track exactly which features are included in different versions, indicated by the minor and patch version numbers.

[UNRELEASED]

No user facing changes.

3.27.7 - 10 Dec 2024

  • We are rolling out a change in December 2024 that will extract the CodeQL bundle directly to the toolcache to improve performance. #2631
  • Update default CodeQL bundle version to 2.20.0. #2636

3.27.6 - 03 Dec 2024

  • Update default CodeQL bundle version to 2.19.4. #2626

3.27.5 - 19 Nov 2024

No user facing changes.

3.27.4 - 14 Nov 2024

No user facing changes.

3.27.3 - 12 Nov 2024

No user facing changes.

3.27.2 - 12 Nov 2024

  • Fixed an issue where setting up the CodeQL tools would sometimes fail with the message "Invalid value 'undefined' for header 'authorization'". #2590

3.27.1 - 08 Nov 2024

  • The CodeQL Action now downloads bundles compressed using Zstandard on GitHub Enterprise Server when using Linux or macOS runners. This speeds up the installation of the CodeQL tools. This feature is already available to GitHub.com users. #2573
  • Update default CodeQL bundle version to 2.19.3. #2576

3.27.0 - 22 Oct 2024

  • Bump the minimum CodeQL bundle version to 2.14.6. #2549
  • Fix an issue where the upload-sarif Action would fail with "upload-sarif post-action step failed: Input required and not supplied: token" when called in a composite Action that had a different set of inputs to the ones expected by the upload-sarif Action. #2557
  • Update default CodeQL bundle version to 2.19.2. #2552

3.26.13 - 14 Oct 2024

No user facing changes.

... (truncated)

Commits
  • babb554 Merge pull request #2640 from github/update-v3.27.7-89757925c
  • 0a5a1c0 Update changelog for v3.27.7
  • 8975792 Merge pull request #2637 from github/dependabot/npm_and_yarn/npm-3bf4e64efa
  • d853bec Update checked-in dependencies
  • aab3460 Bump the npm group with 4 updates
  • 0d3e640 Merge pull request #2636 from github/update-bundle/codeql-bundle-v2.20.0
  • b135154 Merge branch 'main' into update-bundle/codeql-bundle-v2.20.0
  • 3d09005 Add changelog note
  • 8ba1205 Update default bundle to codeql-bundle-v2.20.0
  • 6f9e628 Merge pull request #2634 from github/angelapwen/stop-using-artifact-flag
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github/codeql-action&package-manager=github_actions&previous-version=3.27.6&new-version=3.27.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- .github/workflows/scorecard.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1f0352c52..271b0d75a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,17 +44,17 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/init@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 with: languages: ${{ matrix.language }} # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/autobuild@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/analyze@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7b95c5405..89f460138 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -60,6 +60,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v2.2.4 + uses: github/codeql-action/upload-sarif@babb554ede22fd5605947329c4d04d8e7a0b8155 # v2.2.4 with: sarif_file: results.sarif From a7989f79d35aca0e83e6303bd662ddbf7254c44c Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Wed, 11 Dec 2024 08:13:48 -0600 Subject: [PATCH 18/29] chore: add return types to untyped functions (#1560) ## Description Adds typing to untyped functions End to End Test: (See [Pepr Excellent Examples](https://github.com/defenseunicorns/pepr-excellent-examples)) ## Related Issue Fixes #1551 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- src/lib/assets/deploy.ts | 12 ++++++------ src/lib/assets/destroy.ts | 2 +- src/lib/assets/helm.test.ts | 10 ++++++++-- src/lib/assets/helm.ts | 12 ++++++------ src/lib/assets/index.ts | 10 +++++----- src/lib/assets/pods.test.ts | 24 ++++++++++++------------ src/lib/assets/pods.ts | 15 ++++++++++----- src/lib/assets/webhooks.ts | 2 +- src/lib/assets/yaml.ts | 21 ++++++++++++--------- 9 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/lib/assets/deploy.ts b/src/lib/assets/deploy.ts index 3f933b482..884d07988 100644 --- a/src/lib/assets/deploy.ts +++ b/src/lib/assets/deploy.ts @@ -9,7 +9,7 @@ import { V1PolicyRule as PolicyRule } from "@kubernetes/client-node"; import { Assets } from "."; import Log from "../telemetry/logger"; import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking"; -import { deployment, moduleSecret, namespace, watcher } from "./pods"; +import { getDeployment, getModuleSecret, getNamespace, getWatcher } from "./pods"; import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac"; import { peprStoreCRD } from "./store"; import { webhookConfig } from "./webhooks"; @@ -19,7 +19,7 @@ export async function deployImagePullSecret(imagePullSecret: ImagePullSecret, na try { await K8s(kind.Namespace).Get("pepr-system"); } catch { - await K8s(kind.Namespace).Apply(namespace()); + await K8s(kind.Namespace).Apply(getNamespace()); } try { @@ -48,7 +48,7 @@ export async function deploy(assets: Assets, force: boolean, webhookTimeout?: nu const { name, host, path } = assets; Log.info("Applying pepr-system namespace"); - await K8s(kind.Namespace).Apply(namespace(assets.config.customLabels?.namespace)); + await K8s(kind.Namespace).Apply(getNamespace(assets.config.customLabels?.namespace)); // Create the mutating webhook configuration if it is needed const mutateWebhook = await webhookConfig(assets, "mutate", webhookTimeout); @@ -123,7 +123,7 @@ async function setupController(assets: Assets, code: Buffer, hash: string, force const { name } = assets; Log.info("Applying module secret"); - const mod = moduleSecret(name, code, hash); + const mod = getModuleSecret(name, code, hash); await K8s(kind.Secret).Apply(mod, { force }); Log.info("Applying controller service"); @@ -139,14 +139,14 @@ async function setupController(assets: Assets, code: Buffer, hash: string, force await K8s(kind.Secret).Apply(apiToken, { force }); Log.info("Applying deployment"); - const dep = deployment(assets, hash, assets.buildTimestamp); + const dep = getDeployment(assets, hash, assets.buildTimestamp); await K8s(kind.Deployment).Apply(dep, { force }); } // Setup the watcher deployment and service async function setupWatcher(assets: Assets, hash: string, force: boolean) { // If the module has a watcher, deploy it - const watchDeployment = watcher(assets, hash, assets.buildTimestamp); + const watchDeployment = getWatcher(assets, hash, assets.buildTimestamp); if (watchDeployment) { Log.info("Applying watcher deployment"); await K8s(kind.Deployment).Apply(watchDeployment, { force }); diff --git a/src/lib/assets/destroy.ts b/src/lib/assets/destroy.ts index 6d4a80960..e528e70ff 100644 --- a/src/lib/assets/destroy.ts +++ b/src/lib/assets/destroy.ts @@ -6,7 +6,7 @@ import { K8s, kind } from "kubernetes-fluent-client"; import Log from "../telemetry/logger"; import { peprStoreCRD } from "./store"; -export async function destroyModule(name: string) { +export async function destroyModule(name: string): Promise { const namespace = "pepr-system"; Log.info("Destroying Pepr module"); diff --git a/src/lib/assets/helm.test.ts b/src/lib/assets/helm.test.ts index 172cbcf41..386982e8d 100644 --- a/src/lib/assets/helm.test.ts +++ b/src/lib/assets/helm.test.ts @@ -1,12 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { nsTemplate, chartYaml, watcherDeployTemplate, admissionDeployTemplate, serviceMonitorTemplate } from "./helm"; +import { + namespaceTemplate, + chartYaml, + watcherDeployTemplate, + admissionDeployTemplate, + serviceMonitorTemplate, +} from "./helm"; import { expect, describe, test } from "@jest/globals"; describe("Kubernetes Template Generators", () => { describe("nsTemplate", () => { test("should generate a Namespace template correctly", () => { - const result = nsTemplate(); + const result = namespaceTemplate(); expect(result).toContain("apiVersion: v1"); expect(result).toContain("kind: Namespace"); expect(result).toContain("name: pepr-system"); diff --git a/src/lib/assets/helm.ts b/src/lib/assets/helm.ts index 3f7124aa2..2b65f368a 100644 --- a/src/lib/assets/helm.ts +++ b/src/lib/assets/helm.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -export function clusterRoleTemplate() { +export function clusterRoleTemplate(): string { return ` apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -15,7 +15,7 @@ export function clusterRoleTemplate() { `; } -export function nsTemplate() { +export function namespaceTemplate(): string { return ` apiVersion: v1 kind: Namespace @@ -32,7 +32,7 @@ export function nsTemplate() { `; } -export function chartYaml(name: string, description?: string) { +export function chartYaml(name: string, description?: string): string { return ` apiVersion: v2 name: ${name} @@ -61,7 +61,7 @@ export function chartYaml(name: string, description?: string) { `; } -export function watcherDeployTemplate(buildTimestamp: string) { +export function watcherDeployTemplate(buildTimestamp: string): string { return ` apiVersion: apps/v1 kind: Deployment @@ -142,7 +142,7 @@ export function watcherDeployTemplate(buildTimestamp: string) { `; } -export function admissionDeployTemplate(buildTimestamp: string) { +export function admissionDeployTemplate(buildTimestamp: string): string { return ` apiVersion: apps/v1 kind: Deployment @@ -228,7 +228,7 @@ export function admissionDeployTemplate(buildTimestamp: string) { `; } -export function serviceMonitorTemplate(name: string) { +export function serviceMonitorTemplate(name: string): string { return ` {{- if .Values.${name}.serviceMonitor.enabled }} apiVersion: monitoring.coreos.com/v1 diff --git a/src/lib/assets/index.ts b/src/lib/assets/index.ts index 78d0947d8..b72df1254 100644 --- a/src/lib/assets/index.ts +++ b/src/lib/assets/index.ts @@ -16,7 +16,7 @@ import { dedent } from "../helpers"; import { resolve } from "path"; import { chartYaml, - nsTemplate, + namespaceTemplate, admissionDeployTemplate, watcherDeployTemplate, clusterRoleTemplate, @@ -25,7 +25,7 @@ import { import { promises as fs } from "fs"; import { webhookConfig } from "./webhooks"; import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking"; -import { watcher, moduleSecret } from "./pods"; +import { getWatcher, getModuleSecret } from "./pods"; import { clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac"; import { createDirectoryIfNotExists } from "../filesystemService"; @@ -157,7 +157,7 @@ export class Assets { const pairs: [string, () => string][] = [ [helm.files.chartYaml, (): string => dedent(chartYaml(this.config.uuid, this.config.description || ""))], - [helm.files.namespaceYaml, (): string => dedent(nsTemplate())], + [helm.files.namespaceYaml, (): string => dedent(namespaceTemplate())], [helm.files.watcherServiceYaml, (): string => toYaml(watcherService(this.name))], [helm.files.admissionServiceYaml, (): string => toYaml(service(this.name))], [helm.files.tlsSecretYaml, (): string => toYaml(tlsSecret(this.name, this.tls))], @@ -167,7 +167,7 @@ export class Assets { [helm.files.clusterRoleYaml, (): string => dedent(clusterRoleTemplate())], [helm.files.clusterRoleBindingYaml, (): string => toYaml(clusterRoleBinding(this.name))], [helm.files.serviceAccountYaml, (): string => toYaml(serviceAccount(this.name))], - [helm.files.moduleSecretYaml, (): string => toYaml(moduleSecret(this.name, code, this.hash))], + [helm.files.moduleSecretYaml, (): string => toYaml(getModuleSecret(this.name, code, this.hash))], ]; await Promise.all(pairs.map(async ([file, content]) => await fs.writeFile(file, content()))); @@ -191,7 +191,7 @@ export class Assets { await fs.writeFile(helm.files.validationWebhookYaml, createWebhookYaml(this, validateWebhook)); } - const watchDeployment = watcher(this, this.hash, this.buildTimestamp); + const watchDeployment = getWatcher(this, this.hash, this.buildTimestamp); if (watchDeployment) { await fs.writeFile(helm.files.watcherDeploymentYaml, dedent(watcherDeployTemplate(this.buildTimestamp))); await fs.writeFile(helm.files.watcherServiceMonitorYaml, dedent(serviceMonitorTemplate("watcher"))); diff --git a/src/lib/assets/pods.test.ts b/src/lib/assets/pods.test.ts index a21b4882e..aba42161c 100644 --- a/src/lib/assets/pods.test.ts +++ b/src/lib/assets/pods.test.ts @@ -1,4 +1,4 @@ -import { namespace, watcher, deployment, moduleSecret, genEnv } from "./pods"; +import { getNamespace, getWatcher, getDeployment, getModuleSecret, genEnv } from "./pods"; import { expect, describe, test, jest, afterEach } from "@jest/globals"; import { Assets } from "."; import { ModuleConfig } from "../module"; @@ -296,7 +296,7 @@ const assets: Assets = JSON.parse(`{ }`); describe("namespace function", () => { test("should create a namespace object without labels if none are provided", () => { - const result = namespace(); + const result = getNamespace(); expect(result).toEqual({ apiVersion: "v1", kind: "Namespace", @@ -304,7 +304,7 @@ describe("namespace function", () => { name: "pepr-system", }, }); - const result1 = namespace({ one: "two" }); + const result1 = getNamespace({ one: "two" }); expect(result1).toEqual({ apiVersion: "v1", kind: "Namespace", @@ -318,20 +318,20 @@ describe("namespace function", () => { }); test("should create a namespace object with empty labels if an empty object is provided", () => { - const result = namespace({}); - expect(result.metadata.labels).toEqual({}); + const result = getNamespace({}); + expect(result.metadata?.labels).toEqual({}); }); test("should create a namespace object with provided labels", () => { const labels = { "pepr.dev/controller": "admission", "istio-injection": "enabled" }; - const result = namespace(labels); - expect(result.metadata.labels).toEqual(labels); + const result = getNamespace(labels); + expect(result.metadata?.labels).toEqual(labels); }); }); describe("watcher function", () => { test("watcher with bindings", () => { - const result = watcher(assets, "test-hash", "test-timestamp"); + const result = getWatcher(assets, "test-hash", "test-timestamp"); expect(result).toBeTruthy(); expect(result!.metadata!.name).toBe("pepr-static-test-watcher"); @@ -339,14 +339,14 @@ describe("watcher function", () => { test("watcher without bindings", () => { assets.capabilities = []; - const result = watcher(assets, "test-hash", "test-timestamp"); + const result = getWatcher(assets, "test-hash", "test-timestamp"); expect(result).toBeNull(); }); }); describe("deployment function", () => { test("deployment", () => { - const result = deployment(assets, "test-hash", "test-timestamp"); + const result = getDeployment(assets, "test-hash", "test-timestamp"); expect(result).toBeTruthy(); expect(result!.metadata!.name).toBe("pepr-static-test"); @@ -368,7 +368,7 @@ describe("moduleSecret function", () => { // eslint-disable-next-line @typescript-eslint/no-var-requires jest.spyOn(require("../helpers"), "secretOverLimit").mockReturnValue(false); - const result = moduleSecret(name, data, hash); + const result = getModuleSecret(name, data, hash); expect(result).toEqual({ apiVersion: "v1", @@ -399,7 +399,7 @@ describe("moduleSecret function", () => { throw new Error("process.exit"); }); - expect(() => moduleSecret(name, data, hash)).toThrow("process.exit"); + expect(() => getModuleSecret(name, data, hash)).toThrow("process.exit"); expect(consoleErrorMock).toHaveBeenCalledWith( "Uncaught Exception:", diff --git a/src/lib/assets/pods.ts b/src/lib/assets/pods.ts index 803127b8f..247d95cbb 100644 --- a/src/lib/assets/pods.ts +++ b/src/lib/assets/pods.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { V1EnvVar } from "@kubernetes/client-node"; +import { KubernetesObject, V1EnvVar } from "@kubernetes/client-node"; import { kind } from "kubernetes-fluent-client"; import { gzipSync } from "zlib"; import { secretOverLimit } from "../helpers"; @@ -10,7 +10,7 @@ import { ModuleConfig } from "../module"; import { Binding } from "../types"; /** Generate the pepr-system namespace */ -export function namespace(namespaceLabels?: Record) { +export function getNamespace(namespaceLabels?: Record): KubernetesObject { if (namespaceLabels) { return { apiVersion: "v1", @@ -31,7 +31,12 @@ export function namespace(namespaceLabels?: Record) { } } -export function watcher(assets: Assets, hash: string, buildTimestamp: string, imagePullSecret?: string) { +export function getWatcher( + assets: Assets, + hash: string, + buildTimestamp: string, + imagePullSecret?: string, +): kind.Deployment | null { const { name, image, capabilities, config } = assets; let hasSchedule = false; @@ -186,7 +191,7 @@ export function watcher(assets: Assets, hash: string, buildTimestamp: string, im return deploy; } -export function deployment( +export function getDeployment( assets: Assets, hash: string, buildTimestamp: string, @@ -336,7 +341,7 @@ export function deployment( return deploy; } -export function moduleSecret(name: string, data: Buffer, hash: string): kind.Secret { +export function getModuleSecret(name: string, data: Buffer, hash: string): kind.Secret { // Compress the data const compressed = gzipSync(data); const path = `module-${hash}.js.gz`; diff --git a/src/lib/assets/webhooks.ts b/src/lib/assets/webhooks.ts index 3d8d45915..3a3187340 100644 --- a/src/lib/assets/webhooks.ts +++ b/src/lib/assets/webhooks.ts @@ -20,7 +20,7 @@ const peprIgnoreLabel: V1LabelSelectorRequirement = { const peprIgnoreNamespaces: string[] = ["kube-system", "pepr-system"]; -export async function generateWebhookRules(assets: Assets, isMutateWebhook: boolean) { +export async function generateWebhookRules(assets: Assets, isMutateWebhook: boolean): Promise { const { config, capabilities } = assets; const rules: V1RuleWithOperations[] = []; diff --git a/src/lib/assets/yaml.ts b/src/lib/assets/yaml.ts index e1faffc0a..2a66a4a7f 100644 --- a/src/lib/assets/yaml.ts +++ b/src/lib/assets/yaml.ts @@ -6,13 +6,16 @@ import crypto from "crypto"; import { promises as fs } from "fs"; import { Assets } from "."; import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking"; -import { deployment, moduleSecret, namespace, watcher } from "./pods"; +import { getDeployment, getModuleSecret, getNamespace, getWatcher } from "./pods"; import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac"; import { webhookConfig } from "./webhooks"; import { genEnv } from "./pods"; // Helm Chart overrides file (values.yaml) generated from assets -export async function overridesFile({ hash, name, image, config, apiToken, capabilities }: Assets, path: string) { +export async function overridesFile( + { hash, name, image, config, apiToken, capabilities }: Assets, + path: string, +): Promise { const rbacOverrides = clusterRole(name, capabilities, config.rbacMode, config.rbac).rules; const overrides = { @@ -166,7 +169,7 @@ export async function overridesFile({ hash, name, image, config, apiToken, capab await fs.writeFile(path, dumpYaml(overrides, { noRefs: true, forceQuotes: true })); } -export function zarfYaml({ name, image, config }: Assets, path: string) { +export function zarfYaml({ name, image, config }: Assets, path: string): string { const zarfCfg = { kind: "ZarfPackageConfig", metadata: { @@ -194,7 +197,7 @@ export function zarfYaml({ name, image, config }: Assets, path: string) { return dumpYaml(zarfCfg, { noRefs: true }); } -export function zarfYamlChart({ name, image, config }: Assets, path: string) { +export function zarfYamlChart({ name, image, config }: Assets, path: string): string { const zarfCfg = { kind: "ZarfPackageConfig", metadata: { @@ -223,7 +226,7 @@ export function zarfYamlChart({ name, image, config }: Assets, path: string) { return dumpYaml(zarfCfg, { noRefs: true }); } -export async function allYaml(assets: Assets, imagePullSecret?: string) { +export async function allYaml(assets: Assets, imagePullSecret?: string): Promise { const { name, tls, apiToken, path, config } = assets; const code = await fs.readFile(path); @@ -232,19 +235,19 @@ export async function allYaml(assets: Assets, imagePullSecret?: string) { const mutateWebhook = await webhookConfig(assets, "mutate", assets.config.webhookTimeout); const validateWebhook = await webhookConfig(assets, "validate", assets.config.webhookTimeout); - const watchDeployment = watcher(assets, assets.hash, assets.buildTimestamp, imagePullSecret); + const watchDeployment = getWatcher(assets, assets.hash, assets.buildTimestamp, imagePullSecret); const resources = [ - namespace(assets.config.customLabels?.namespace), + getNamespace(assets.config.customLabels?.namespace), clusterRole(name, assets.capabilities, config.rbacMode, config.rbac), clusterRoleBinding(name), serviceAccount(name), apiTokenSecret(name, apiToken), tlsSecret(name, tls), - deployment(assets, assets.hash, assets.buildTimestamp, imagePullSecret), + getDeployment(assets, assets.hash, assets.buildTimestamp, imagePullSecret), service(name), watcherService(name), - moduleSecret(name, code, assets.hash), + getModuleSecret(name, code, assets.hash), storeRole(name), storeRoleBinding(name), ]; From 0e50003556cedef66e464241634ebc4a0a6da337 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Wed, 11 Dec 2024 12:42:15 -0500 Subject: [PATCH 19/29] chore: complexity in build (#1557) ## Description Build.ts ```bash 74:13 warning Async arrow function has too many statements (54). Maximum allowed is 20 max-statements 74:13 warning Async arrow function has a complexity of 18. Maximum allowed is 10 complexity 251:8 warning Async function 'buildModule' has too many statements (35). Maximum allowed is 20 max-statements ``` ## Related Issue Fixes #1539 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --------- Signed-off-by: Case Wylie Co-authored-by: Sam Mayer --- src/cli/build.helpers.ts | 180 ++++++++++++++++++++++++++ src/cli/build.test.ts | 268 ++++++++++++++++++++++++++++++++++++++- src/cli/build.ts | 218 +++++++++++++------------------ 3 files changed, 527 insertions(+), 139 deletions(-) diff --git a/src/cli/build.helpers.ts b/src/cli/build.helpers.ts index 217913943..fa7a5126a 100644 --- a/src/cli/build.helpers.ts +++ b/src/cli/build.helpers.ts @@ -1,3 +1,15 @@ +import { createDirectoryIfNotExists } from "../lib/filesystemService"; +import { sanitizeResourceName } from "../sdk/sdk"; +import { createDockerfile } from "../lib/included-files"; +import { execSync } from "child_process"; +import { CapabilityExport } from "../lib/types"; +import { validateCapabilityNames } from "../lib/helpers"; +import { BuildOptions, BuildResult, context, BuildContext } from "esbuild"; +import { Assets } from "../lib/assets"; +import { resolve } from "path"; +import { promises as fs } from "fs"; + +export type Reloader = (opts: BuildResult) => void | Promise; /** * Determine the RBAC mode based on the CLI options and the module's config * @param opts CLI options @@ -26,3 +38,171 @@ export function determineRbacMode( // if nothing is defined return admin, else return scoped return cfg.pepr.rbacMode || "admin"; } + +/** + * Handle the custom output directory + * @param outputDir the desired output directory + * @returns The desired output directory or the default one + */ + +export async function handleCustomOutputDir(outputDir: string): Promise { + const defaultOutputDir = "dist"; + if (outputDir) { + try { + await createDirectoryIfNotExists(outputDir); + return outputDir; + } catch (error) { + console.error(`Error creating output directory: ${error.message}`); + process.exit(1); + } + } + return defaultOutputDir; +} + +/** + * Check if the image is from Iron Bank and return the correct image + * @param registry The registry of the image + * @param image The image to check + * @param peprVersion The version of the PEPR controller + * @returns The image string + * @example + */ +export function checkIronBankImage(registry: string, image: string, peprVersion: string): string { + return registry === "Iron Bank" + ? `registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller:v${peprVersion}` + : image; +} + +/** + * Check if the image pull secret is a valid Kubernetes name + * @param imagePullSecret + * @returns boolean + */ +export function validImagePullSecret(imagePullSecretName: string): void { + if (imagePullSecretName) { + const error = "Invalid imagePullSecret. Please provide a valid name as defined in RFC 1123."; + if (sanitizeResourceName(imagePullSecretName) !== imagePullSecretName) { + // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names + console.error(error); + process.exit(1); + } + } +} + +/** + * Constraint to majke sure customImage and registry are not both used + * @param customImage + * @param registry + * @returns + */ +export function handleCustomImage(customImage: string, registry: string): string { + let defaultImage = ""; + if (customImage) { + if (registry) { + console.error(`Custom Image and registry cannot be used together.`); + process.exit(1); + } + defaultImage = customImage; + } + return defaultImage; +} + +/** + * Creates and pushes a custom image for WASM or any other included files + * @param includedFiles + * @param peprVersion + * @param description + * @param image + */ +export async function handleCustomImageBuild( + includedFiles: string[], + peprVersion: string, + description: string, + image: string, +): Promise { + if (includedFiles.length > 0) { + await createDockerfile(peprVersion, description, includedFiles); + execSync(`docker build --tag ${image} -f Dockerfile.controller .`, { + stdio: "inherit", + }); + execSync(`docker push ${image}`, { stdio: "inherit" }); + } +} + +/** + * Disables embedding of deployment files into output module + * @param embed + * @param path + * @returns + */ +export function handleEmbedding(embed: boolean, path: string): void { + if (!embed) { + console.info(`✅ Module built successfully at ${path}`); + return; + } +} + +/** + * Check if the capability names are valid + * @param capabilities The capabilities to check + */ +export function handleValidCapabilityNames(capabilities: CapabilityExport[]): void { + try { + // wait for capabilities to be loaded and test names + validateCapabilityNames(capabilities); + } catch (e) { + console.error(`Error loading capability:`, e); + process.exit(1); + } +} + +/** + * Watch for changes in the module + * @param ctxCfg The build options + * @param reloader The reloader function + * @returns The build context + */ +export async function watchForChanges( + ctxCfg: BuildOptions, + reloader: Reloader | undefined, +): Promise> { + const ctx = await context(ctxCfg); + + // If the reloader function is defined, watch the module for changes + if (reloader) { + await ctx.watch(); + } else { + // Otherwise, just build the module once + await ctx.rebuild(); + await ctx.dispose(); + } + + return ctx; +} + +export async function generateYamlAndWriteToDisk(obj: { + uuid: string; + imagePullSecret: string; + outputDir: string; + assets: Assets; + zarf: string; +}): Promise { + const { uuid, imagePullSecret, outputDir, assets, zarf } = obj; + const yamlFile = `pepr-module-${uuid}.yaml`; + const chartPath = `${uuid}-chart`; + const yamlPath = resolve(outputDir, yamlFile); + const yaml = await assets.allYaml(imagePullSecret); + const zarfPath = resolve(outputDir, "zarf.yaml"); + + let localZarf = ""; + if (zarf === "chart") { + localZarf = assets.zarfYamlChart(chartPath); + } else { + localZarf = assets.zarfYaml(yamlFile); + } + await fs.writeFile(yamlPath, yaml); + await fs.writeFile(zarfPath, localZarf); + + await assets.generateHelmChart(outputDir); + console.info(`✅ K8s resource for the module saved to ${yamlPath}`); +} diff --git a/src/cli/build.test.ts b/src/cli/build.test.ts index 1c310b55a..d0afc3de0 100644 --- a/src/cli/build.test.ts +++ b/src/cli/build.test.ts @@ -1,36 +1,292 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { determineRbacMode } from "./build.helpers"; +import { + determineRbacMode, + handleCustomOutputDir, + handleEmbedding, + handleValidCapabilityNames, + handleCustomImageBuild, + checkIronBankImage, + validImagePullSecret, + handleCustomImage, +} from "./build.helpers"; +import { createDirectoryIfNotExists } from "../lib/filesystemService"; +import { expect, describe, it, jest, beforeEach } from "@jest/globals"; +import { createDockerfile } from "../lib/included-files"; +import { execSync } from "child_process"; +import { CapabilityExport } from "../lib/types"; +import { Capability } from "../lib/capability"; -import { expect, describe, test } from "@jest/globals"; +jest.mock("child_process", () => ({ + execSync: jest.fn(), +})); + +jest.mock("../lib/included-files", () => ({ + createDockerfile: jest.fn(), +})); + +jest.mock("../lib/filesystemService", () => ({ + createDirectoryIfNotExists: jest.fn(), +})); describe("determineRbacMode", () => { - test("should allow CLI options to overwrite module config", () => { + it("should allow CLI options to overwrite module config", () => { const opts = { rbacMode: "admin" }; const cfg = { pepr: { rbacMode: "scoped" } }; const result = determineRbacMode(opts, cfg); expect(result).toBe("admin"); }); - test('should return "admin" when cfg.pepr.rbacMode is provided and not "scoped"', () => { + it('should return "admin" when cfg.pepr.rbacMode is provided and not "scoped"', () => { const opts = {}; const cfg = { pepr: { rbacMode: "admin" } }; const result = determineRbacMode(opts, cfg); expect(result).toBe("admin"); }); - test('should return "scoped" when cfg.pepr.rbacMode is "scoped"', () => { + it('should return "scoped" when cfg.pepr.rbacMode is "scoped"', () => { const opts = {}; const cfg = { pepr: { rbacMode: "scoped" } }; const result = determineRbacMode(opts, cfg); expect(result).toBe("scoped"); }); - test("should default to admin when neither option is provided", () => { + it("should default to admin when neither option is provided", () => { const opts = {}; const cfg = { pepr: {} }; const result = determineRbacMode(opts, cfg); expect(result).toBe("admin"); }); }); + +describe("handleCustomOutputDir", () => { + const mockedCreateDirectoryIfNotExists = jest.mocked(createDirectoryIfNotExists); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should return the provided output directory if it exists and is created successfully", async () => { + mockedCreateDirectoryIfNotExists.mockResolvedValueOnce(); + + const outputDir = "custom-output-dir"; + const result = await handleCustomOutputDir(outputDir); + + expect(mockedCreateDirectoryIfNotExists).toHaveBeenCalledWith(outputDir); + expect(result).toBe(outputDir); + }); + + it("should return the default output directory if no custom directory is provided", async () => { + const outputDir = ""; + const result = await handleCustomOutputDir(outputDir); + expect(result).toBe("dist"); + }); +}); + +describe("checkIronBankImage", () => { + it("should return the Iron Bank image if the registry is Iron Bank", () => { + const registry = "Iron Bank"; + const image = "ghcr.io/defenseunicorns/pepr/controller:v0.0.1"; + const peprVersion = "0.0.1"; + const result = checkIronBankImage(registry, image, peprVersion); + expect(result).toBe( + `registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller:v${peprVersion}`, + ); + }); + + it("should return the image if the registry is not Iron Bank", () => { + const registry = "GitHub"; + const image = "ghcr.io/defenseunicorns/pepr/controller:v0.0.1"; + const peprVersion = "0.0.1"; + const result = checkIronBankImage(registry, image, peprVersion); + expect(result).toBe(image); + }); +}); + +describe("validImagePullSecret", () => { + const mockExit = jest.spyOn(process, "exit").mockImplementation(() => { + return undefined as never; + }); + + const consoleErrorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + + beforeEach(() => { + jest.clearAllMocks(); + }); + it("should not throw an error if the imagePullSecret is valid", () => { + const imagePullSecret = "valid-secret"; + validImagePullSecret(imagePullSecret); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + it("should not throw an error if the imagePullSecret is empty", () => { + const imagePullSecret = ""; + validImagePullSecret(imagePullSecret); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + it("should throw an error if the imagePullSecret is invalid", () => { + const imagePullSecret = "invalid name"; + validImagePullSecret(imagePullSecret); + expect(consoleErrorSpy).toHaveBeenCalled(); + expect(mockExit).toHaveBeenCalled(); + }); +}); +describe("handleCustomImage", () => { + const mockExit = jest.spyOn(process, "exit").mockImplementation(() => { + return undefined as never; + }); + + const consoleErrorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should return the customImage if no registry is provided", () => { + const customImage = "custom-image"; + const registry = ""; + + const result = handleCustomImage(customImage, registry); + + expect(result).toBe(customImage); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + + it("should return an empty string if neither customImage nor registry is provided", () => { + const customImage = ""; + const registry = ""; + + const result = handleCustomImage(customImage, registry); + + expect(result).toBe(""); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + + it("should call process.exit with 1 and log an error if both customImage and registry are provided", () => { + const customImage = "custom-image"; + const registry = "registry"; + + handleCustomImage(customImage, registry); + expect(consoleErrorSpy).toHaveBeenCalledWith( + "Custom Image and registry cannot be used together.", + ); + expect(mockExit).toHaveBeenCalledWith(1); + }); +}); + +describe("handleCustomImageBuild", () => { + const mockedExecSync = jest.mocked(execSync); + const mockedCreateDockerfile = jest.mocked(createDockerfile); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should call createDockerfile and execute docker commands if includedFiles is not empty", async () => { + const includedFiles = ["file1", "file2"]; + const peprVersion = "1.0.0"; + const description = "Test Description"; + const image = "test-image"; + + await handleCustomImageBuild(includedFiles, peprVersion, description, image); + + expect(mockedCreateDockerfile).toHaveBeenCalledWith(peprVersion, description, includedFiles); + expect(mockedExecSync).toHaveBeenCalledWith( + `docker build --tag ${image} -f Dockerfile.controller .`, + { + stdio: "inherit", + }, + ); + expect(mockedExecSync).toHaveBeenCalledWith(`docker push ${image}`, { stdio: "inherit" }); + }); + + it("should not call createDockerfile or execute docker commands if includedFiles is empty", async () => { + const includedFiles: string[] = []; + const peprVersion = "1.0.0"; + const description = "Test Description"; + const image = "test-image"; + + await handleCustomImageBuild(includedFiles, peprVersion, description, image); + + expect(mockedCreateDockerfile).not.toHaveBeenCalled(); + expect(mockedExecSync).not.toHaveBeenCalled(); + }); +}); +describe("handleEmbedding", () => { + const consoleInfoSpy = jest.spyOn(console, "info").mockImplementation(() => {}); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should log success message if embed is false", () => { + const embed = false; + const path = "test/path"; + + handleEmbedding(embed, path); + + expect(consoleInfoSpy).toHaveBeenCalledWith(`✅ Module built successfully at ${path}`); + }); + + it("should not log success message if embed is true", () => { + const embed = true; + const path = "test/path"; + + handleEmbedding(embed, path); + + expect(consoleInfoSpy).not.toHaveBeenCalled(); + }); +}); + +describe("handleValidCapabilityNames", () => { + const mockExit = jest.spyOn(process, "exit").mockImplementation(() => { + return undefined as never; + }); + + const consoleErrorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + + it("should call validateCapabilityNames with capabilities", () => { + const capability = new Capability({ + name: "test", + description: "test", + }); + + const capabilityExports: CapabilityExport[] = [ + { + name: capability.name, + description: capability.description, + namespaces: capability.namespaces, + bindings: capability.bindings, + hasSchedule: capability.hasSchedule, + }, + ]; + + handleValidCapabilityNames(capabilityExports); + expect(consoleErrorSpy).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + it("should call validateCapabilityNames with capabilities", () => { + const capability = new Capability({ + name: "test $me", + description: "test", + }); + + const capabilityExports: CapabilityExport[] = [ + { + name: capability.name, + description: capability.description, + namespaces: capability.namespaces, + bindings: capability.bindings, + hasSchedule: capability.hasSchedule, + }, + ]; + + handleValidCapabilityNames(capabilityExports); + expect(consoleErrorSpy).toHaveBeenCalled(); + expect(mockExit).toHaveBeenCalled(); + }); +}); diff --git a/src/cli/build.ts b/src/cli/build.ts index 5ec58740e..c1eabd90f 100644 --- a/src/cli/build.ts +++ b/src/cli/build.ts @@ -1,25 +1,34 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { execSync, execFileSync } from "child_process"; -import { BuildOptions, BuildResult, analyzeMetafile, context } from "esbuild"; +import { execFileSync } from "child_process"; +import { BuildOptions, BuildResult, analyzeMetafile } from "esbuild"; import { promises as fs } from "fs"; import { basename, dirname, extname, resolve } from "path"; -import { createDockerfile } from "../lib/included-files"; import { Assets } from "../lib/assets"; import { dependencies, version } from "./init/templates"; import { RootCmd } from "./root"; -import { peprFormat } from "./format"; import { Option } from "commander"; -import { validateCapabilityNames, parseTimeout } from "../lib/helpers"; -import { sanitizeResourceName } from "../sdk/sdk"; -import { determineRbacMode } from "./build.helpers"; -import { createDirectoryIfNotExists } from "../lib/filesystemService"; +import { parseTimeout } from "../lib/helpers"; +import { peprFormat } from "./format"; +import { + watchForChanges, + determineRbacMode, + handleEmbedding, + handleCustomOutputDir, + handleValidCapabilityNames, + handleCustomImage, + handleCustomImageBuild, + checkIronBankImage, + validImagePullSecret, + generateYamlAndWriteToDisk, +} from "./build.helpers"; + const peprTS = "pepr.ts"; let outputDir: string = "dist"; export type Reloader = (opts: BuildResult) => void | Promise; -export default function (program: RootCmd) { +export default function (program: RootCmd): void { program .command("build") .description("Build a Pepr Module for deployment") @@ -73,13 +82,7 @@ export default function (program: RootCmd) { ) .action(async opts => { // assign custom output directory if provided - if (opts.outputDir) { - outputDir = opts.outputDir; - createDirectoryIfNotExists(outputDir).catch(error => { - console.error(`Error creating output directory: ${error.message}`); - process.exit(1); - }); - } + outputDir = await handleCustomOutputDir(opts.outputDir); // Build the module const buildModuleResult = await buildModule(undefined, opts.entryPoint, opts.embed); @@ -88,16 +91,7 @@ export default function (program: RootCmd) { // Files to include in controller image for WASM support const { includedFiles } = cfg.pepr; - let image: string = ""; - - // Build Kubernetes manifests with custom image - if (opts.customImage) { - if (opts.registry) { - console.error(`Custom Image and registry cannot be used together.`); - process.exit(1); - } - image = opts.customImage; - } + let image = handleCustomImage(opts.customImage, opts.registry); // Check if there is a custom timeout defined if (opts.timeout !== undefined) { @@ -111,25 +105,14 @@ export default function (program: RootCmd) { image = `${opts.registryInfo}/custom-pepr-controller:${cfg.pepr.peprVersion}`; // only actually build/push if there are files to include - if (includedFiles.length > 0) { - await createDockerfile(cfg.pepr.peprVersion, cfg.description, includedFiles); - execSync(`docker build --tag ${image} -f Dockerfile.controller .`, { - stdio: "inherit", - }); - execSync(`docker push ${image}`, { stdio: "inherit" }); - } + await handleCustomImageBuild(includedFiles, cfg.pepr.peprVersion, cfg.description, image); } // If building without embedding, exit after building - if (!opts.embed) { - console.info(`✅ Module built successfully at ${path}`); - return; - } + handleEmbedding(opts.embed, path); // set the image version if provided - if (opts.version) { - cfg.pepr.peprVersion = opts.version; - } + opts.version ? (cfg.pepr.peprVersion = opts.version) : null; // Generate a secret for the module const assets = new Assets( @@ -144,56 +127,22 @@ export default function (program: RootCmd) { ); // If registry is set to Iron Bank, use Iron Bank image - if (opts?.registry === "Iron Bank") { - console.info( - `\n\tThis command assumes the latest release. Pepr's Iron Bank image release cycle is dictated by renovate and is typically released a few days after the GitHub release.\n\tAs an alternative you may consider custom --custom-image to target a specific image and version.`, - ); - image = `registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller:v${cfg.pepr.peprVersion}`; - } + image = checkIronBankImage(opts.registry, image, cfg.pepr.peprVersion); // if image is a custom image, use that instead of the default - if (image !== "") { - assets.image = image; - } + image !== "" ? (assets.image = image) : null; // Ensure imagePullSecret is valid - if (opts.withPullSecret) { - if (sanitizeResourceName(opts.withPullSecret) !== opts.withPullSecret) { - // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names - console.error( - "Invalid imagePullSecret. Please provide a valid name as defined in RFC 1123.", - ); - process.exit(1); - } - } - - const yamlFile = `pepr-module-${uuid}.yaml`; - const chartPath = `${uuid}-chart`; - const yamlPath = resolve(outputDir, yamlFile); - const yaml = await assets.allYaml(opts.withPullSecret); - - try { - // wait for capabilities to be loaded and test names - validateCapabilityNames(assets.capabilities); - } catch (e) { - console.error(`Error loading capability:`, e); - process.exit(1); - } - - const zarfPath = resolve(outputDir, "zarf.yaml"); - - let zarf = ""; - if (opts.zarf === "chart") { - zarf = assets.zarfYamlChart(chartPath); - } else { - zarf = assets.zarfYaml(yamlFile); - } - await fs.writeFile(yamlPath, yaml); - await fs.writeFile(zarfPath, zarf); - - await assets.generateHelmChart(outputDir); - - console.info(`✅ K8s resource for the module saved to ${yamlPath}`); + validImagePullSecret(opts.withPullSecret); + + handleValidCapabilityNames(assets.capabilities); + await generateYamlAndWriteToDisk({ + uuid, + outputDir, + imagePullSecret: opts.withPullSecret, + zarf: opts.zarf, + assets, + }); } }); } @@ -252,15 +201,7 @@ export async function buildModule(reloader?: Reloader, entryPoint = peprTS, embe try { const { cfg, modulePath, path, uuid } = await loadModule(entryPoint); - const validFormat = await peprFormat(true); - - if (!validFormat) { - console.log( - "\x1b[33m%s\x1b[0m", - "Formatting errors were found. The build will continue, but you may want to run `npx pepr format` to address any issues.", - ); - } - + await checkFormat(); // Resolve node_modules folder (in support of npm workspaces!) const npmRoot = execFileSync("npm", ["root"]).toString().trim(); @@ -282,7 +223,7 @@ export async function buildModule(reloader?: Reloader, entryPoint = peprTS, embe plugins: [ { name: "reload-server", - setup(build) { + setup(build): void | Promise { build.onEnd(async r => { // Print the build size analysis if (r?.metafile) { @@ -322,53 +263,64 @@ export async function buildModule(reloader?: Reloader, entryPoint = peprTS, embe ctxCfg.treeShaking = false; } - const ctx = await context(ctxCfg); - - // If the reloader function is defined, watch the module for changes - if (reloader) { - await ctx.watch(); - } else { - // Otherwise, just build the module once - await ctx.rebuild(); - await ctx.dispose(); - } + const ctx = await watchForChanges(ctxCfg, reloader); return { ctx, path, cfg, uuid }; } catch (e) { - console.error(`Error building module:`, e); + handleModuleBuildError(e); + } +} - if (!e.stdout) process.exit(1); // Exit with a non-zero exit code on any other error +interface BuildModuleResult { + stdout?: Buffer; + stderr: Buffer; +} - const out = e.stdout.toString() as string; - const err = e.stderr.toString(); +function handleModuleBuildError(e: BuildModuleResult): void { + console.error(`Error building module:`, e); - console.log(out); - console.error(err); + if (!e.stdout) process.exit(1); // Exit with a non-zero exit code on any other error - // Check for version conflicts - if (out.includes("Types have separate declarations of a private property '_name'.")) { - // Try to find the conflicting package - const pgkErrMatch = /error TS2322: .*? 'import\("\/.*?\/node_modules\/(.*?)\/node_modules/g; - out.matchAll(pgkErrMatch); + const out = e.stdout.toString() as string; + const err = e.stderr.toString(); - // Look for package conflict errors - const conflicts = [...out.matchAll(pgkErrMatch)]; + console.log(out); + console.error(err); - // If the regex didn't match, leave a generic error - if (conflicts.length < 1) { - console.info( - `\n\tOne or more imported Pepr Capabilities seem to be using an incompatible version of Pepr.\n\tTry updating your Pepr Capabilities to their latest versions.`, - "Version Conflict", - ); - } + // Check for version conflicts + if (out.includes("Types have separate declarations of a private property '_name'.")) { + // Try to find the conflicting package + const pgkErrMatch = /error TS2322: .*? 'import\("\/.*?\/node_modules\/(.*?)\/node_modules/g; + out.matchAll(pgkErrMatch); - // Otherwise, loop through each conflicting package and print an error - conflicts.forEach(match => { - console.info( - `\n\tPackage '${match[1]}' seems to be incompatible with your current version of Pepr.\n\tTry updating to the latest version.`, - "Version Conflict", - ); - }); + // Look for package conflict errors + const conflicts = [...out.matchAll(pgkErrMatch)]; + + // If the regex didn't match, leave a generic error + if (conflicts.length < 1) { + console.info( + `\n\tOne or more imported Pepr Capabilities seem to be using an incompatible version of Pepr.\n\tTry updating your Pepr Capabilities to their latest versions.`, + "Version Conflict", + ); } + + // Otherwise, loop through each conflicting package and print an error + conflicts.forEach(match => { + console.info( + `\n\tPackage '${match[1]}' seems to be incompatible with your current version of Pepr.\n\tTry updating to the latest version.`, + "Version Conflict", + ); + }); + } +} + +export async function checkFormat() { + const validFormat = await peprFormat(true); + + if (!validFormat) { + console.log( + "\x1b[33m%s\x1b[0m", + "Formatting errors were found. The build will continue, but you may want to run `npx pepr format` to address any issues.", + ); } } From ac05c1d2050f2606b72c005196ba55bebedf6cba Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Wed, 11 Dec 2024 13:55:07 -0600 Subject: [PATCH 20/29] chore: add return types to watch-processor.ts (#1562) ## Description Adds return typing to untyped functions. ## Related Issue Fixes #1546 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- src/lib/watch-processor.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/lib/watch-processor.ts b/src/lib/watch-processor.ts index 4458d92f4..1f36a9c9c 100644 --- a/src/lib/watch-processor.ts +++ b/src/lib/watch-processor.ts @@ -20,7 +20,7 @@ const queues: Record> = {}; * @param obj The object to derive a key from * @returns The key to a Queue in the list of queues */ -export function queueKey(obj: KubernetesObject) { +export function queueKey(obj: KubernetesObject): string { const options = ["kind", "kindNs", "kindNsName", "global"]; const d3fault = "kind"; @@ -40,7 +40,7 @@ export function queueKey(obj: KubernetesObject) { return lookup[strat]; } -export function getOrCreateQueue(obj: KubernetesObject) { +export function getOrCreateQueue(obj: KubernetesObject): Queue { const key = queueKey(obj); if (!queues[key]) { queues[key] = new Queue(key); @@ -74,7 +74,7 @@ const eventToPhaseMap = { * * @param capabilities The capabilities to load watches for */ -export function setupWatch(capabilities: Capability[], ignoredNamespaces?: string[]) { +export function setupWatch(capabilities: Capability[], ignoredNamespaces?: string[]): void { capabilities.map(capability => capability.bindings .filter(binding => binding.isWatch) @@ -88,14 +88,18 @@ export function setupWatch(capabilities: Capability[], ignoredNamespaces?: strin * @param binding the binding to watch * @param capabilityNamespaces list of namespaces to filter on */ -async function runBinding(binding: Binding, capabilityNamespaces: string[], ignoredNamespaces?: string[]) { +async function runBinding( + binding: Binding, + capabilityNamespaces: string[], + ignoredNamespaces?: string[], +): Promise { // Get the phases to match, fallback to any const phaseMatch: WatchPhase[] = eventToPhaseMap[binding.event] || eventToPhaseMap[Event.ANY]; // The watch callback is run when an object is received or dequeued Log.debug({ watchCfg }, "Effective WatchConfig"); - const watchCallback = async (kubernetesObject: KubernetesObject, phase: WatchPhase) => { + const watchCallback = async (kubernetesObject: KubernetesObject, phase: WatchPhase): Promise => { // First, filter the object based on the phase if (phaseMatch.includes(phase)) { try { @@ -117,7 +121,7 @@ async function runBinding(binding: Binding, capabilityNamespaces: string[], igno } }; - const handleFinalizerRemoval = async (kubernetesObject: KubernetesObject) => { + const handleFinalizerRemoval = async (kubernetesObject: KubernetesObject): Promise => { if (!kubernetesObject.metadata?.deletionTimestamp) { return; } @@ -191,7 +195,7 @@ async function runBinding(binding: Binding, capabilityNamespaces: string[], igno } } -export function logEvent(event: WatchEvent, message: string = "", obj?: KubernetesObject) { +export function logEvent(event: WatchEvent, message: string = "", obj?: KubernetesObject): void { const logMessage = `Watch event ${event} received${message ? `. ${message}.` : "."}`; if (obj) { Log.debug(obj, logMessage); From 9e77f669d93aee5e4fdc65075535c0d24cb6e0cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:30:38 -0500 Subject: [PATCH 21/29] chore: bump @types/node from 22.10.1 to 22.10.2 in the development-dependencies group (#1565) Bumps the development-dependencies group with 1 update: [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node). Updates `@types/node` from 22.10.1 to 22.10.2
Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | @types/node | [>= 20.a, < 21] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@types/node&package-manager=npm_and_yarn&previous-version=22.10.1&new-version=22.10.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a7379b312..09b1f1a9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2686,9 +2686,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", - "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "version": "22.10.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", "license": "MIT", "dependencies": { "undici-types": "~6.20.0" From 65ea7d1da920ec056e4368a4f31eefd69fc6ffb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:30:54 +0000 Subject: [PATCH 22/29] chore: bump trufflesecurity/trufflehog from 3.86.0 to 3.86.1 (#1564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.86.0 to 3.86.1.
Release notes

Sourced from trufflesecurity/trufflehog's releases.

v3.86.1

What's Changed

Full Changelog: https://github.com/trufflesecurity/trufflehog/compare/v3.86.0...v3.86.1

Commits
  • 6ceb490 fix(deps): update module golang.org/x/crypto to v0.31.0 (#3767)
  • bce9610 fix(deps): update golang.org/x/exp digest to 1829a12 (#3761)
  • 3525c6f [refactor] - s3 metrics (#3760)
  • c24b4d5 [Fix] detector's integration tests starting with alphabet 'g' (#3765)
  • b26aa17 [Fix] detector's integration tests starting with alphabet 'e' & 'f' (#3764)
  • 2821fb3 updated testingbot detector and it's integration tests (#3763)
  • e716a84 fix(deps): update module google.golang.org/api to v0.211.0 (#3759)
  • fa18638 fix(deps): update golang.org/x/exp digest to 1443442 (#3758)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=trufflesecurity/trufflehog&package-manager=github_actions&previous-version=3.86.0&new-version=3.86.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/secret-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index da2e4771a..671df9f02 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -23,6 +23,6 @@ jobs: with: fetch-depth: 0 - name: Default Secret Scanning - uses: trufflesecurity/trufflehog@f726d02330dbcec836fa17f79fa7711fdb3a5cc8 # main + uses: trufflesecurity/trufflehog@6ceb49097f21249369f015c4d571173e9252f04d # main with: extra_args: --debug --no-verification # Warn on potential violations From 6a26b80c9ea0ab332b83edef643851924e3b6df8 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 12 Dec 2024 12:22:28 -0500 Subject: [PATCH 23/29] chore: different periods between send and receive (#1563) ## Description We lowered the debounce period (to one second) in which our Store Subscribers **receive** updates from the Watch on the `PeprStore` resource, meaning they can react to changes faster. The same debounce period was responsible for **sending** JSON patches to the `PeprStore` resource. The cache that holds these updates is updated and the items that were sent are deleted after the JSON patch succeeds. This meant that if we are sending events every second, and the kube-apiserver does not process the patch in a second, then the same events will be sent over multiple intervals. It is unclear if this would make the data incorrect as it is idempotent but it certainly introduces pressure on the network which could create a fallout of other network created problems. This PR: - Creates two different debounce periods, one for send and one for receive - Sends a reason for the reject in `SetItemAndWait/RemoveItemAndWait` - Cleans up the timeouts in `SetItemAndWait/RemoveItemAndWait` - Adds an array item to the JSON patch to the Store to ensure a watch event occurs to resolve promises in Set/RemoveItemAndWait e2e: https://github.com/defenseunicorns/pepr-excellent-examples/pull/204 ## Related Issue Fixes #1561 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --------- Signed-off-by: Case Wylie --- src/lib/controller/store.ts | 19 +++++++-- src/lib/controller/storeCache.test.ts | 19 ++++++++- src/lib/controller/storeCache.ts | 11 +++++- src/lib/schedule.test.ts | 8 ++-- src/lib/storage.ts | 57 ++++++++++++++++----------- 5 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/lib/controller/store.ts b/src/lib/controller/store.ts index 159ff8a7e..2289da8ce 100644 --- a/src/lib/controller/store.ts +++ b/src/lib/controller/store.ts @@ -12,7 +12,8 @@ import { DataOp, DataSender, DataStore, Storage } from "../storage"; import { fillStoreCache, sendUpdatesAndFlushCache } from "./storeCache"; const namespace = "pepr-system"; -export const debounceBackoff = 1000; +const debounceBackoffReceive = 1000; +const debounceBackoffSend = 4000; export class StoreController { #name: string; @@ -68,6 +69,15 @@ export class StoreController { #migrateAndSetupWatch = async (store: Store): Promise => { Log.debug(redactedStore(store), "Pepr Store migration"); + // Add cacheID label to store + await K8s(Store, { namespace, name: this.#name }).Patch([ + { + op: "add", + path: "/metadata/labels/pepr.dev-cacheID", + value: `${Date.now()}`, + }, + ]); + const data: DataStore = store.data || {}; let storeCache: Record = {}; @@ -134,7 +144,7 @@ export class StoreController { // Debounce the update to 1 second to avoid multiple rapid calls clearTimeout(this.#sendDebounce); - this.#sendDebounce = setTimeout(debounced, this.#onReady ? 0 : debounceBackoff); + this.#sendDebounce = setTimeout(debounced, this.#onReady ? 0 : debounceBackoffReceive); }; #send = (capabilityName: string): DataSender => { @@ -151,7 +161,7 @@ export class StoreController { Log.debug(redactedPatch(storeCache), "Sending updates to Pepr store"); void sendUpdatesAndFlushCache(storeCache, namespace, this.#name); } - }, debounceBackoff); + }, debounceBackoffSend); return sender; }; @@ -165,6 +175,9 @@ export class StoreController { metadata: { name: this.#name, namespace, + labels: { + "pepr.dev-cacheID": `${Date.now()}`, + }, }, data: { // JSON Patch will die if the data is empty, so we need to add a placeholder diff --git a/src/lib/controller/storeCache.test.ts b/src/lib/controller/storeCache.test.ts index 5e4ddb96d..3594bea55 100644 --- a/src/lib/controller/storeCache.test.ts +++ b/src/lib/controller/storeCache.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, jest, afterEach } from "@jest/globals"; -import { fillStoreCache, sendUpdatesAndFlushCache } from "./storeCache"; +import { fillStoreCache, sendUpdatesAndFlushCache, updateCacheID } from "./storeCache"; import { Operation } from "fast-json-patch"; import { GenericClass, K8s, KubernetesObject } from "kubernetes-fluent-client"; import { K8sInit } from "kubernetes-fluent-client/dist/fluent/types"; @@ -100,3 +100,20 @@ describe("sendCache", () => { }); }); }); + +describe("updateCacheId", () => { + it("should update the metadata label of the cacheID in the payload array of patches", () => { + const patches: Operation[] = [ + { + op: "add", + path: "/data/hello-pepr-v2-a", + value: "a", + }, + ]; + + const updatedPatches = updateCacheID(patches); + expect(updatedPatches.length).toBe(2); + expect(updatedPatches[1].op).toBe("replace"); + expect(updatedPatches[1].path).toBe("/metadata/labels/pepr.dev-cacheID"); + }); +}); diff --git a/src/lib/controller/storeCache.ts b/src/lib/controller/storeCache.ts index 50340f107..fb27ea886 100644 --- a/src/lib/controller/storeCache.ts +++ b/src/lib/controller/storeCache.ts @@ -11,7 +11,7 @@ export const sendUpdatesAndFlushCache = async (cache: Record, try { if (payload.length > 0) { - await K8s(Store, { namespace, name }).Patch(payload); // Send patch to cluster + await K8s(Store, { namespace, name }).Patch(updateCacheID(payload)); // Send patch to cluster Object.keys(cache).forEach(key => delete cache[key]); } } catch (err) { @@ -61,3 +61,12 @@ export const fillStoreCache = ( } return cache; }; + +export function updateCacheID(payload: Operation[]): Operation[] { + payload.push({ + op: "replace", + path: "/metadata/labels/pepr.dev-cacheID", + value: `${Date.now()}`, + }); + return payload; +} diff --git a/src/lib/schedule.test.ts b/src/lib/schedule.test.ts index cafdb0179..ca84784a4 100644 --- a/src/lib/schedule.test.ts +++ b/src/lib/schedule.test.ts @@ -20,10 +20,10 @@ export class MockStorage { this.storage[key] = value; } - setItemAndWait(key: string, value: string): Promise { + setItemAndWait(key: string, value: string): Promise { return new Promise(resolve => { this.storage[key] = value; - resolve(); + resolve("ok"); }); } @@ -31,10 +31,10 @@ export class MockStorage { delete this.storage[key]; } - removeItemAndWait(key: string): Promise { + removeItemAndWait(key: string): Promise { return new Promise(resolve => { delete this.storage[key]; - resolve(); + resolve("ok"); }); } diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 2f56b2f12..5e26f4a8b 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -12,6 +12,11 @@ export type Unsubscribe = () => void; const MAX_WAIT_TIME = 15000; const STORE_VERSION_PREFIX = "v2"; +interface WaitRecord { + timeout?: ReturnType; + unsubscribe?: () => void; +} + export function v2StoreKey(key: string): string { return `${STORE_VERSION_PREFIX}-${pointer.escape(key)}`; } @@ -58,13 +63,13 @@ export interface PeprStore { * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously. * Resolves when the key/value show up in the store. */ - setItemAndWait(key: string, value: string): Promise; + setItemAndWait(key: string, value: string): Promise; /** * Remove the value of the key. * Resolves when the key does not show up in the store. */ - removeItemAndWait(key: string): Promise; + removeItemAndWait(key: string): Promise; } /** @@ -128,22 +133,24 @@ export class Storage implements PeprStore { * @param value - The value of the key * @returns */ - setItemAndWait = (key: string, value: string): Promise => { + setItemAndWait = (key: string, value: string): Promise => { this.#dispatchUpdate("add", [v2StoreKey(key)], value); + const record: WaitRecord = {}; - return new Promise((resolve, reject) => { - const unsubscribe = this.subscribe(data => { + return new Promise((resolve, reject) => { + // If promise has not resolved before MAX_WAIT_TIME reject + record.timeout = setTimeout(() => { + record.unsubscribe!(); + return reject(`MAX_WAIT_TIME elapsed: Key ${key} not seen in ${MAX_WAIT_TIME / 1000}s`); + }, MAX_WAIT_TIME); + + record.unsubscribe = this.subscribe(data => { if (data[`${v2UnescapedStoreKey(key)}`] === value) { - unsubscribe(); - resolve(); + record.unsubscribe!(); + clearTimeout(record.timeout); + resolve("ok"); } }); - - // If promise has not resolved before MAX_WAIT_TIME reject - setTimeout(() => { - unsubscribe(); - return reject(); - }, MAX_WAIT_TIME); }); }; @@ -154,21 +161,23 @@ export class Storage implements PeprStore { * @param key - The key to add into the store * @returns */ - removeItemAndWait = (key: string): Promise => { + removeItemAndWait = (key: string): Promise => { this.#dispatchUpdate("remove", [v2StoreKey(key)]); - return new Promise((resolve, reject) => { - const unsubscribe = this.subscribe(data => { + const record: WaitRecord = {}; + return new Promise((resolve, reject) => { + // If promise has not resolved before MAX_WAIT_TIME reject + record.timeout = setTimeout(() => { + record.unsubscribe!(); + return reject(`MAX_WAIT_TIME elapsed: Key ${key} still seen after ${MAX_WAIT_TIME / 1000}s`); + }, MAX_WAIT_TIME); + + record.unsubscribe = this.subscribe(data => { if (!Object.hasOwn(data, `${v2UnescapedStoreKey(key)}`)) { - unsubscribe(); - resolve(); + record.unsubscribe!(); + clearTimeout(record.timeout); + resolve("ok"); } }); - - // If promise has not resolved before MAX_WAIT_TIME reject - setTimeout(() => { - unsubscribe(); - return reject(); - }, MAX_WAIT_TIME); }); }; From 46c851499aa51f05a61d628b8348a45729331a69 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 12 Dec 2024 13:43:04 -0500 Subject: [PATCH 24/29] chore: roadmap 2025 (#1544) ## Description Start the 2025 roadmap, collaborate with team. ## Related Issue Fixes #1485 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --------- Signed-off-by: Case Wylie Co-authored-by: Barrett <81570928+btlghrants@users.noreply.github.com> Co-authored-by: Sam Mayer --- docs/090_roadmap/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/090_roadmap/README.md b/docs/090_roadmap/README.md index fccb78f8d..a27feeb8d 100644 --- a/docs/090_roadmap/README.md +++ b/docs/090_roadmap/README.md @@ -66,3 +66,38 @@ _2024 Roadmap_ - Load test Pepr/KFC to identify bottlenecks and areas of improvement. - Ensure that Pepr/KFC can handle a large number of resources and events over a sustained period of time (nightly). +_2025 Roadmap_ +## Phase 1: Code Quality - Experimentation + +- **Q1**: + - **Turn on eslint enforcement and configure settings and see no warnings**: + - Eliminate circular dependencies, complexity, return statements, etc. + - **Metric and Performance Baselining**: + - Establish a baseline for performance and resource utilization metrics. Use this data to make informed decisions about the direction of the project in terms of Deno2 + - **OTEL Preparation**: + - Come up with a plan to implement Open Telemetry. Specifically distributed tracing, metrics, logs and events. Use this data to make debugging easier from a UDS Core prespective. There will be documentation work on how to use an OTEL collector with a Pepr Module. + - **Nightly Release**: + - Establish a nightly release process. This will help us to catch bugs early and ensure that the project is always in a releasable state. + +## Phase 2: Durable Storage for Metrics and Performance Tests / Transactional Pepr Store + +- **Q2**: + - **Professional Dashboard displaying metrics and performance tests originating from CI**: + - **Determine if a Transactional PeprStore makes sense**: + - Sus out details involved with having a transactional Pepr Store. What are the implications of this? What are the benefits? What are the drawbacks? What are the use-cases? What are the technologies that can be used to implement this? + - **Experimentation with Deno2**: + - Experiment with Deno2 through Dash Days and see if it can be used in the project. Look into the performance improvements and new features that Deno2 brings to the table. + + +## Phase 3: TBD + +- **Q3**: + - **Deno2 Implementation**: + - If determined to be advisable, move forward with migrating the project to Deno2 (starting with the kubernetes-fluent-client..?). This phase will focus on adapting the codebase, conducting extensive testing, and creating comprehensive documentation to ensure a seamless transition. + - **Transactional PeprStore Implementation**: + - Begin integrating transactional functionality into PeprStore. The implementation will emphasize robust testing and clear documentation to support fast and reliable data operations in a transactional manner. + +## Phase 4: TDB + +- **Q4**: + From 672937001807d131aefb3422908c0f2864e28748 Mon Sep 17 00:00:00 2001 From: Barrett <81570928+btlghrants@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:27:16 -0600 Subject: [PATCH 25/29] refactor: resolve eslint warnings (max-depth, complexity) - src/lib/mutate-processor.ts (#1543) ### Describe what should be investigated or refactored Refactor of `src/lib/mutate-processor.ts` to reduce complexity warnings: ``` /pepr/src/lib/mutate-processor.ts 18:8 warning Async function 'mutateProcessor' has a complexity of 18. Maximum allowed is 10 complexity 18:8 warning Async function 'mutateProcessor' has too many statements (49). Maximum allowed is 20 max-statements 99:9 warning Blocks are nested too deeply (4). Maximum allowed is 3 max-depth ``` ### Additional context Fixes #1532 In support of #1248 --- src/lib/mutate-processor.test.ts | 294 +++++++++++++++++++++++++++++++ src/lib/mutate-processor.ts | 258 ++++++++++++++++----------- 2 files changed, 453 insertions(+), 99 deletions(-) create mode 100644 src/lib/mutate-processor.test.ts diff --git a/src/lib/mutate-processor.test.ts b/src/lib/mutate-processor.test.ts new file mode 100644 index 000000000..30645cced --- /dev/null +++ b/src/lib/mutate-processor.test.ts @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present The Pepr Authors + +import { beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { clone } from "ramda"; +import { ModuleConfig } from "./module"; +import { PeprMutateRequest } from "./mutate-request"; +import * as sut from "./mutate-processor"; +import { AdmissionRequest, Binding, MutateAction } from "./types"; +import { Event, Operation } from "./enums"; +import { convertFromBase64Map, convertToBase64Map } from "./utils"; +import { GenericClass, KubernetesObject } from "kubernetes-fluent-client"; +import { MutateResponse } from "./k8s"; +import { Errors } from "./errors"; + +jest.mock("./utils"); +const mockConvertFromBase64Map = jest.mocked(convertFromBase64Map); +const mockConvertToBase64Map = jest.mocked(convertToBase64Map); + +const defaultModuleConfig: ModuleConfig = { + uuid: "test-uuid", + alwaysIgnore: {}, +}; + +const defaultAdmissionRequest: AdmissionRequest = { + uid: "uid", + kind: { + kind: "kind", + group: "group", + version: "version", + }, + resource: { + group: "group", + version: "version", + resource: "resource", + }, + name: "", + object: { + metadata: { + name: "create-me", + }, + }, + operation: Operation.CREATE, + userInfo: {}, +}; + +const defaultPeprMutateRequest = (admissionRequest = defaultAdmissionRequest) => + new PeprMutateRequest(admissionRequest); + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe("updateStatus", () => { + describe("when given non-delete request", () => { + it("adds status annotation to to-be-admitted resource", () => { + const name = "capa"; + const status = "test-status"; + const annote = `${defaultModuleConfig.uuid}.pepr.dev/${name}`; + + const result = sut.updateStatus(defaultModuleConfig, name, defaultPeprMutateRequest(), status); + + expect(result.HasAnnotation(annote)).toBe(true); + expect(result.Raw.metadata?.annotations?.[annote]).toBe(status); + }); + }); + + describe("when given delete request", () => { + it("does not add status annotation to to-be-admitted resource", () => { + const testAdmissionRequest = { + ...clone(defaultAdmissionRequest), + operation: Operation.DELETE, + oldObject: {}, + }; + const name = "capa"; + const annote = `${defaultModuleConfig.uuid}.pepr.dev/${name}`; + + const result = sut.updateStatus( + defaultModuleConfig, + name, + defaultPeprMutateRequest(testAdmissionRequest), + "test-status", + ); + + expect(result.HasAnnotation(annote)).toBe(false); + }); + }); +}); + +describe("logMutateErrorMessage", () => { + it.each([ + // error msg, result string + ["oof", "oof"], + ["", "An error occurred with the mutate action."], + ["[object Object]", "An error occurred with the mutate action."], + ])("given error '%s', returns '%s'", (err, res) => { + const result = sut.logMutateErrorMessage(new Error(err)); + expect(result).toBe(res); + }); +}); + +describe("decodeData", () => { + const skips = ["convert", "From", "Base64", "Map"]; + + beforeEach(() => { + mockConvertFromBase64Map.mockImplementation(() => skips); + }); + + it("returns skips if required & given a Secret", () => { + const testAdmissionRequest = { + ...defaultAdmissionRequest, + kind: { + kind: "Secret", + version: "v1", + group: "", + }, + }; + const testPeprMutateRequest = defaultPeprMutateRequest(testAdmissionRequest); + + const { skipped, wrapped } = sut.decodeData(testPeprMutateRequest); + + expect(mockConvertFromBase64Map.mock.calls.length).toBe(1); + expect(mockConvertFromBase64Map.mock.calls[0].at(0)).toBe(testPeprMutateRequest.Raw); + expect(skipped).toBe(skips); + expect(wrapped).toBe(testPeprMutateRequest); + }); + + it("returns no skips when given a non-Secret", () => { + const testAdmissionRequest = { + ...defaultAdmissionRequest, + kind: { + kind: "NotASecret", + version: "v1", + group: "", + }, + }; + const testPeprMutateRequest = defaultPeprMutateRequest(testAdmissionRequest); + + const { skipped, wrapped } = sut.decodeData(testPeprMutateRequest); + + expect(mockConvertFromBase64Map.mock.calls.length).toBe(0); + expect(skipped).toEqual([]); + expect(wrapped).toBe(testPeprMutateRequest); + }); +}); + +describe("reencodeData", () => { + it("returns unchanged content when given non-secret", () => { + const skipped = ["convert", "To", "Base64", "Map"]; + const testAdmissionRequest = { + ...defaultAdmissionRequest, + kind: { + kind: "NotASecret", + version: "v1", + group: "", + }, + }; + const testPeprMutateRequest = defaultPeprMutateRequest(testAdmissionRequest); + + const transformed = sut.reencodeData(testPeprMutateRequest, skipped); + + expect(mockConvertToBase64Map.mock.calls.length).toBe(0); + expect(transformed).toEqual(testAdmissionRequest.object); + }); + + it("returns modified content when given a secret and skips", () => { + const skipped = ["convert", "To", "Base64", "Map"]; + const testAdmissionRequest = { + ...defaultAdmissionRequest, + kind: { + kind: "Secret", + version: "v1", + group: "", + }, + }; + const testPeprMutateRequest = defaultPeprMutateRequest(testAdmissionRequest); + + const transformed = sut.reencodeData(testPeprMutateRequest, skipped); + + expect(mockConvertToBase64Map.mock.calls.length).toBe(1); + expect(mockConvertToBase64Map.mock.calls[0].at(0)).toEqual(testPeprMutateRequest.Raw); + expect(mockConvertToBase64Map.mock.calls[0].at(1)).toBe(skipped); + expect(transformed).toEqual(testPeprMutateRequest.Raw); + }); +}); + +const defaultBinding: Binding = { + event: Event.CREATE, + model: {} as GenericClass, + kind: { + kind: "kind", + group: "group", + version: "version", + }, + filters: { + annotations: {}, + deletionTimestamp: false, + labels: {}, + name: "", + namespaces: [], + regexName: "", + regexNamespaces: [], + }, + mutateCallback: jest.fn() as jest.Mocked>, +}; + +const defaultBindable: sut.Bindable = { + req: defaultAdmissionRequest, + config: defaultModuleConfig, + name: "test-name", + namespaces: [], + binding: defaultBinding, + actMeta: {}, +}; + +const defaultMutateResponse: MutateResponse = { + uid: "default-uid", + allowed: true, +}; + +describe("processRequest", () => { + it("adds a status annotation on success", async () => { + const testPeprMutateRequest = defaultPeprMutateRequest(); + const testMutateResponse = clone(defaultMutateResponse); + const annote = `${defaultModuleConfig.uuid}.pepr.dev/${defaultBindable.name}`; + + const result = await sut.processRequest(defaultBindable, testPeprMutateRequest, testMutateResponse); + + expect(result).toEqual({ wrapped: testPeprMutateRequest, response: testMutateResponse }); + expect(result.wrapped.Raw.metadata?.annotations).toBeDefined(); + expect(result.wrapped.Raw.metadata!.annotations![annote]).toBe("succeeded"); + + expect(result.response.warnings).toBeUndefined(); + expect(result.response.result).toBeUndefined(); + expect(result.response.auditAnnotations).toBeUndefined(); + }); + + it("adds a status annotation, warning, and result on failure when Errors.reject", async () => { + const mutateCallback = (jest.fn() as jest.Mocked>).mockImplementation( + () => { + throw "oof"; + }, + ); + const testBinding = { ...clone(defaultBinding), mutateCallback }; + const testBindable = { ...clone(defaultBindable), binding: testBinding }; + testBindable.config.onError = Errors.reject; + const testPeprMutateRequest = defaultPeprMutateRequest(); + const testMutateResponse = clone(defaultMutateResponse); + const annote = `${defaultModuleConfig.uuid}.pepr.dev/${defaultBindable.name}`; + + const result = await sut.processRequest(testBindable, testPeprMutateRequest, testMutateResponse); + + expect(result).toEqual({ wrapped: testPeprMutateRequest, response: testMutateResponse }); + expect(result.wrapped.Raw.metadata?.annotations).toBeDefined(); + expect(result.wrapped.Raw.metadata!.annotations![annote]).toBe("warning"); + + expect(result.response.warnings).toHaveLength(1); + expect(result.response.warnings![0]).toBe("Action failed: An error occurred with the mutate action."); + expect(result.response.result).toBe("Pepr module configured to reject on error"); + expect(result.response.auditAnnotations).toBeUndefined(); + }); + + it("adds a status annotation, warning, and auditAnnotation on failure when Errors.audit", async () => { + const mutateCallback = (jest.fn() as jest.Mocked>).mockImplementation( + () => { + throw "oof"; + }, + ); + const testBinding = { ...clone(defaultBinding), mutateCallback }; + const testBindable = { ...clone(defaultBindable), binding: testBinding }; + testBindable.config.onError = Errors.audit; + const testPeprMutateRequest = defaultPeprMutateRequest(); + const testMutateResponse = clone(defaultMutateResponse); + const annote = `${defaultModuleConfig.uuid}.pepr.dev/${defaultBindable.name}`; + + const result = await sut.processRequest(testBindable, testPeprMutateRequest, testMutateResponse); + + expect(result).toEqual({ wrapped: testPeprMutateRequest, response: testMutateResponse }); + expect(result.wrapped.Raw.metadata?.annotations).toBeDefined(); + expect(result.wrapped.Raw.metadata!.annotations![annote]).toBe("warning"); + + expect(result.response.warnings).toHaveLength(1); + expect(result.response.warnings![0]).toBe("Action failed: An error occurred with the mutate action."); + expect(result.response.result).toBeUndefined(); + expect(result.response.auditAnnotations).toBeDefined(); + + const auditAnnotes = Object.entries(result.response.auditAnnotations!); + expect(auditAnnotes).toHaveLength(1); + + const [key, val] = auditAnnotes[0]; + expect(Date.now() - parseInt(key)).toBeLessThan(5); + expect(val).toBe("Action failed: An error occurred with the mutate action."); + }); +}); diff --git a/src/lib/mutate-processor.ts b/src/lib/mutate-processor.ts index 941a95817..6176a6cd6 100644 --- a/src/lib/mutate-processor.ts +++ b/src/lib/mutate-processor.ts @@ -2,111 +2,187 @@ // SPDX-FileCopyrightText: 2023-Present The Pepr Authors import jsonPatch from "fast-json-patch"; -import { kind } from "kubernetes-fluent-client"; +import { kind, KubernetesObject } from "kubernetes-fluent-client"; +import { clone } from "ramda"; import { Capability } from "./capability"; import { Errors } from "./errors"; import { shouldSkipRequest } from "./filter/filter"; import { MutateResponse } from "./k8s"; -import { AdmissionRequest } from "./types"; +import { AdmissionRequest, Binding } from "./types"; import Log from "./telemetry/logger"; import { ModuleConfig } from "./module"; import { PeprMutateRequest } from "./mutate-request"; import { base64Encode, convertFromBase64Map, convertToBase64Map } from "./utils"; +export interface Bindable { + req: AdmissionRequest; + config: ModuleConfig; + name: string; + namespaces: string[]; + binding: Binding; + actMeta: Record; +} + +export interface Result { + wrapped: PeprMutateRequest; + response: MutateResponse; +} + +// Add annotations to the request to indicate that the capability started processing +// this will allow tracking of failed mutations that were permitted to continue +export function updateStatus( + config: ModuleConfig, + name: string, + wrapped: PeprMutateRequest, + status: string, +): PeprMutateRequest { + // Only update the status if the request is a CREATE or UPDATE (we don't use CONNECT) + if (wrapped.Request.operation === "DELETE") { + return wrapped; + } + wrapped.SetAnnotation(`${config.uuid}.pepr.dev/${name}`, status); + + return wrapped; +} + +export function logMutateErrorMessage(e: Error): string { + try { + if (e.message && e.message !== "[object Object]") { + return e.message; + } else { + throw new Error("An error occurred in the mutate action."); + } + } catch (e) { + return "An error occurred with the mutate action."; + } +} + +export function decodeData(wrapped: PeprMutateRequest): { + skipped: string[]; + wrapped: PeprMutateRequest; +} { + let skipped: string[] = []; + + const isSecret = wrapped.Request.kind.version === "v1" && wrapped.Request.kind.kind === "Secret"; + if (isSecret) { + // convertFromBase64Map modifies it's arg rather than returing a mod'ed copy (ye olde side-effect special, blerg) + skipped = convertFromBase64Map(wrapped.Raw as unknown as kind.Secret); + } + + return { skipped, wrapped }; +} + +export function reencodeData(wrapped: PeprMutateRequest, skipped: string[]): KubernetesObject { + const transformed = clone(wrapped.Raw); + + const isSecret = wrapped.Request.kind.version === "v1" && wrapped.Request.kind.kind === "Secret"; + if (isSecret) { + // convertToBase64Map modifies it's arg rather than returing a mod'ed copy (ye olde side-effect special, blerg) + convertToBase64Map(transformed as unknown as kind.Secret, skipped); + } + + return transformed; +} + +export async function processRequest( + bindable: Bindable, + wrapped: PeprMutateRequest, + response: MutateResponse, +): Promise { + const { binding, actMeta, name, config } = bindable; + + const label = binding.mutateCallback!.name; + Log.info(actMeta, `Processing mutation action (${label})`); + + wrapped = updateStatus(config, name, wrapped, "started"); + + try { + // Run the action + await binding.mutateCallback!(wrapped); + + // Log on success + Log.info(actMeta, `Mutation action succeeded (${label})`); + + // Add annotations to the request to indicate that the capability succeeded + wrapped = updateStatus(config, name, wrapped, "succeeded"); + } catch (e) { + wrapped = updateStatus(config, name, wrapped, "warning"); + response.warnings = response.warnings || []; + + const errorMessage = logMutateErrorMessage(e); + + // Log on failure + Log.error(actMeta, `Action failed: ${errorMessage}`); + response.warnings.push(`Action failed: ${errorMessage}`); + + switch (config.onError) { + case Errors.reject: + response.result = "Pepr module configured to reject on error"; + break; + + case Errors.audit: + response.auditAnnotations = response.auditAnnotations || {}; + response.auditAnnotations[Date.now()] = `Action failed: ${errorMessage}`; + break; + } + } + + return { wrapped, response }; +} + +/* eslint max-statements: ["warn", 25] */ export async function mutateProcessor( config: ModuleConfig, capabilities: Capability[], req: AdmissionRequest, reqMetadata: Record, ): Promise { - const wrapped = new PeprMutateRequest(req); - const response: MutateResponse = { + let response: MutateResponse = { uid: req.uid, warnings: [], allowed: false, }; - // Track whether any capability matched the request - let matchedAction = false; + const decoded = decodeData(new PeprMutateRequest(req)); + let wrapped = decoded.wrapped; - // Track data fields that should be skipped during decoding - let skipDecode: string[] = []; + Log.info(reqMetadata, `Processing request`); - // If the resource is a secret, decode the data - const isSecret = req.kind.version === "v1" && req.kind.kind === "Secret"; - if (isSecret) { - skipDecode = convertFromBase64Map(wrapped.Raw as unknown as kind.Secret); - } + let bindables: Bindable[] = capabilities.flatMap(capa => + capa.bindings.map(bind => ({ + req, + config, + name: capa.name, + namespaces: capa.namespaces, + binding: bind, + actMeta: { ...reqMetadata, name: capa.name }, + })), + ); + + bindables = bindables.filter(bind => { + if (!bind.binding.mutateCallback) { + return false; + } - Log.info(reqMetadata, `Processing request`); + const shouldSkip = shouldSkipRequest( + bind.binding, + bind.req, + bind.namespaces, + bind.config?.alwaysIgnore?.namespaces, + ); + if (shouldSkip !== "") { + Log.debug(shouldSkip); + return false; + } - for (const { name, bindings, namespaces } of capabilities) { - const actionMetadata = { ...reqMetadata, name }; - for (const action of bindings) { - // Skip this action if it's not a mutate action - if (!action.mutateCallback) { - continue; - } - - // Continue to the next action without doing anything if this one should be skipped - const shouldSkip = shouldSkipRequest(action, req, namespaces, config?.alwaysIgnore?.namespaces); - if (shouldSkip !== "") { - Log.debug(shouldSkip); - continue; - } - - const label = action.mutateCallback.name; - Log.info(actionMetadata, `Processing mutation action (${label})`); - matchedAction = true; - - // Add annotations to the request to indicate that the capability started processing - // this will allow tracking of failed mutations that were permitted to continue - const updateStatus = (status: string) => { - // Only update the status if the request is a CREATE or UPDATE (we don't use CONNECT) - if (req.operation === "DELETE") { - return; - } - - const identifier = `${config.uuid}.pepr.dev/${name}`; - wrapped.Raw.metadata = wrapped.Raw.metadata || {}; - wrapped.Raw.metadata.annotations = wrapped.Raw.metadata.annotations || {}; - wrapped.Raw.metadata.annotations[identifier] = status; - }; - - updateStatus("started"); - - try { - // Run the action - await action.mutateCallback(wrapped); - - // Log on success - Log.info(actionMetadata, `Mutation action succeeded (${label})`); - - // Add annotations to the request to indicate that the capability succeeded - updateStatus("succeeded"); - } catch (e) { - updateStatus("warning"); - response.warnings = response.warnings || []; - - const errorMessage = logMutateErrorMessage(e); - - // Log on failure - Log.error(actionMetadata, `Action failed: ${errorMessage}`); - response.warnings.push(`Action failed: ${errorMessage}`); - - switch (config.onError) { - case Errors.reject: - Log.error(actionMetadata, `Action failed: ${errorMessage}`); - response.result = "Pepr module configured to reject on error"; - return response; - - case Errors.audit: - response.auditAnnotations = response.auditAnnotations || {}; - response.auditAnnotations[Date.now()] = `Action failed: ${errorMessage}`; - break; - } - } + return true; + }); + + for (const bindable of bindables) { + ({ wrapped, response } = await processRequest(bindable, wrapped, response)); + if (config.onError === Errors.reject && response?.warnings!.length > 0) { + return response; } } @@ -114,7 +190,7 @@ export async function mutateProcessor( response.allowed = true; // If no capability matched the request, exit early - if (!matchedAction) { + if (bindables.length === 0) { Log.info(reqMetadata, `No matching actions found`); return response; } @@ -124,12 +200,8 @@ export async function mutateProcessor( return response; } - const transformed = wrapped.Raw; - - // Post-process the Secret requests to convert it back to the original format - if (isSecret) { - convertToBase64Map(transformed as unknown as kind.Secret, skipDecode); - } + // unskip base64-encoded data fields that were skipDecode'd + const transformed = reencodeData(wrapped, decoded.skipped); // Compare the original request to the modified request to get the patches const patches = jsonPatch.compare(req.object, transformed); @@ -151,15 +223,3 @@ export async function mutateProcessor( return response; } - -const logMutateErrorMessage = (e: Error): string => { - try { - if (e.message && e.message !== "[object Object]") { - return e.message; - } else { - throw new Error("An error occurred in the mutate action."); - } - } catch (e) { - return "An error occurred with the mutate action."; - } -}; From 0dec56ee2899ebad7248191014b63c6f3b48138a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 08:08:29 -0500 Subject: [PATCH 26/29] chore: bump github/codeql-action from 3.27.7 to 3.27.8 (#1573) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.7 to 3.27.8.
Changelog

Sourced from github/codeql-action's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

Note that the only difference between v2 and v3 of the CodeQL Action is the node version they support, with v3 running on node 20 while we continue to release v2 to support running on node 16. For example 3.22.11 was the first v3 release and is functionally identical to 2.22.11. This approach ensures an easy way to track exactly which features are included in different versions, indicated by the minor and patch version numbers.

[UNRELEASED]

No user facing changes.

3.27.8 - 12 Dec 2024

  • Fixed an issue where streaming the download and extraction of the CodeQL bundle did not respect proxy settings. #2624

3.27.7 - 10 Dec 2024

  • We are rolling out a change in December 2024 that will extract the CodeQL bundle directly to the toolcache to improve performance. #2631
  • Update default CodeQL bundle version to 2.20.0. #2636

3.27.6 - 03 Dec 2024

  • Update default CodeQL bundle version to 2.19.4. #2626

3.27.5 - 19 Nov 2024

No user facing changes.

3.27.4 - 14 Nov 2024

No user facing changes.

3.27.3 - 12 Nov 2024

No user facing changes.

3.27.2 - 12 Nov 2024

  • Fixed an issue where setting up the CodeQL tools would sometimes fail with the message "Invalid value 'undefined' for header 'authorization'". #2590

3.27.1 - 08 Nov 2024

  • The CodeQL Action now downloads bundles compressed using Zstandard on GitHub Enterprise Server when using Linux or macOS runners. This speeds up the installation of the CodeQL tools. This feature is already available to GitHub.com users. #2573
  • Update default CodeQL bundle version to 2.19.3. #2576

3.27.0 - 22 Oct 2024

  • Bump the minimum CodeQL bundle version to 2.14.6. #2549
  • Fix an issue where the upload-sarif Action would fail with "upload-sarif post-action step failed: Input required and not supplied: token" when called in a composite Action that had a different set of inputs to the ones expected by the upload-sarif Action. #2557
  • Update default CodeQL bundle version to 2.19.2. #2552

... (truncated)

Commits
  • 8a93837 Merge pull request #2645 from github/update-v3.27.8-9cfbef4bd
  • 90a2700 Update changelog for v3.27.8
  • 9cfbef4 Merge pull request #2644 from github/aeisenberg/use-app-token-for-release
  • 9a8645d Use an app token for triggering a release
  • 78d0136 Merge pull request #2643 from github/marcogario/robustify_start_proxy_post
  • c4bbe15 Merge pull request #2624 from github/NlightNFotis/detect_use_proxy_when_strea...
  • 47dd68e formatting
  • 849b60e Add token information
  • f327a84 Avoid failing the workflow on a proxy post step
  • 1e5b591 Merge branch 'main' into NlightNFotis/detect_use_proxy_when_streaming
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github/codeql-action&package-manager=github_actions&previous-version=3.27.7&new-version=3.27.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- .github/workflows/scorecard.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 271b0d75a..d7e5355e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,17 +44,17 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 + uses: github/codeql-action/init@8a93837afdf1873301a68d777844b43e98cd4313 # v3.27.8 with: languages: ${{ matrix.language }} # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 + uses: github/codeql-action/autobuild@8a93837afdf1873301a68d777844b43e98cd4313 # v3.27.8 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 + uses: github/codeql-action/analyze@8a93837afdf1873301a68d777844b43e98cd4313 # v3.27.8 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 89f460138..c82707fcb 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -60,6 +60,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@babb554ede22fd5605947329c4d04d8e7a0b8155 # v2.2.4 + uses: github/codeql-action/upload-sarif@8a93837afdf1873301a68d777844b43e98cd4313 # v2.2.4 with: sarif_file: results.sarif From 2829066a100b12587aa284dc1a0dabd75bc63978 Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Fri, 13 Dec 2024 09:39:07 -0600 Subject: [PATCH 27/29] chore: add typing to untyped functions (#1572) ## Description This PR adds typing to the return types of some untyped functions. #1545 mentions `sdk.ts` and `cosign.ts`, but those files are already fully-typed with their returns. End to End Test: (See [Pepr Excellent Examples](https://github.com/defenseunicorns/pepr-excellent-examples)) ## Related Issue Fixes #1545 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- src/runtime/controller.ts | 4 ++-- src/sdk/heredoc.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/controller.ts b/src/runtime/controller.ts index ae30a8a69..c2b0e61fa 100644 --- a/src/runtime/controller.ts +++ b/src/runtime/controller.ts @@ -14,7 +14,7 @@ import { peprStoreCRD } from "../lib/assets/store"; import { validateHash } from "../lib/helpers"; const { version } = packageJSON; -function runModule(expectedHash: string) { +function runModule(expectedHash: string): void { const gzPath = `/app/load/module-${expectedHash}.js.gz`; const jsPath = `/app/module-${expectedHash}.js`; @@ -59,7 +59,7 @@ Log.info(`Pepr Controller (v${version})`); const hash = process.argv[2]; -const startup = async () => { +const startup = async (): Promise => { try { Log.info("Applying the Pepr Store CRD if it doesn't exist"); await K8s(kind.CustomResourceDefinition).Apply(peprStoreCRD, { force: true }); diff --git a/src/sdk/heredoc.ts b/src/sdk/heredoc.ts index c80746f0a..eccadee74 100644 --- a/src/sdk/heredoc.ts +++ b/src/sdk/heredoc.ts @@ -1,7 +1,7 @@ // Refs: // - https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates -export function heredoc(strings: TemplateStringsArray, ...values: string[]) { +export function heredoc(strings: TemplateStringsArray, ...values: string[]): string { // shuffle strings & expression values back together const zipped = strings .reduce((acc: string[], cur, idx) => { From f618e2b255d5d8b3f999f4e01f19d8aca9162385 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 13 Dec 2024 11:13:08 -0500 Subject: [PATCH 28/29] chore: return types on module, included-files, and helpers to standardize our typing (#1574) ## Description Return types on module, included-files, and helpers to standardize our typing ## Related Issue Fixes #1548 Relates to # ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [ ] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [ ] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed Signed-off-by: Case Wylie Co-authored-by: Sam Mayer --- src/lib/helpers.ts | 10 +++++----- src/lib/included-files.ts | 2 +- src/lib/module.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index 597a5d3de..e36663cd2 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -143,7 +143,7 @@ export function filterNoMatchReason( ); } -export function addVerbIfNotExists(verbs: string[], verb: string) { +export function addVerbIfNotExists(verbs: string[], verb: string): void { if (!verbs.includes(verb)) { verbs.push(verb); } @@ -200,11 +200,11 @@ export function hasAnyOverlap(array1: T[], array2: T[]): boolean { return array1.some(element => array2.includes(element)); } -export function ignoredNamespaceConflict(ignoreNamespaces: string[], bindingNamespaces: string[]) { +export function ignoredNamespaceConflict(ignoreNamespaces: string[], bindingNamespaces: string[]): boolean { return hasAnyOverlap(bindingNamespaces, ignoreNamespaces); } -export function bindingAndCapabilityNSConflict(bindingNamespaces: string[], capabilityNamespaces: string[]) { +export function bindingAndCapabilityNSConflict(bindingNamespaces: string[], capabilityNamespaces: string[]): boolean { if (!capabilityNamespaces) { return false; } @@ -215,7 +215,7 @@ export function generateWatchNamespaceError( ignoredNamespaces: string[], bindingNamespaces: string[], capabilityNamespaces: string[], -) { +): string { let err = ""; // check if binding uses an ignored namespace @@ -237,7 +237,7 @@ export function generateWatchNamespaceError( } // namespaceComplianceValidator ensures that capability bindings respect ignored and capability namespaces -export function namespaceComplianceValidator(capability: CapabilityExport, ignoredNamespaces?: string[]) { +export function namespaceComplianceValidator(capability: CapabilityExport, ignoredNamespaces?: string[]): void { const { namespaces: capabilityNamespaces, bindings, name } = capability; const bindingNamespaces: string[] = bindings.flatMap((binding: Binding) => binding.filters.namespaces); const bindingRegexNamespaces: string[] = bindings.flatMap( diff --git a/src/lib/included-files.ts b/src/lib/included-files.ts index 836ab49c9..7ad3373ba 100644 --- a/src/lib/included-files.ts +++ b/src/lib/included-files.ts @@ -3,7 +3,7 @@ import { promises as fs } from "fs"; -export async function createDockerfile(version: string, description: string, includedFiles: string[]) { +export async function createDockerfile(version: string, description: string, includedFiles: string[]): Promise { const file = ` # Use an official Node.js runtime as the base image FROM ghcr.io/defenseunicorns/pepr/controller:v${version} diff --git a/src/lib/module.ts b/src/lib/module.ts index f7949ef24..17c5f4085 100644 --- a/src/lib/module.ts +++ b/src/lib/module.ts @@ -58,12 +58,12 @@ export type PeprModuleOptions = { }; // Track if this is a watch mode controller -export const isWatchMode = () => process.env.PEPR_WATCH_MODE === "true"; +export const isWatchMode = (): boolean => process.env.PEPR_WATCH_MODE === "true"; // Track if Pepr is running in build mode -export const isBuildMode = () => process.env.PEPR_MODE === "build"; +export const isBuildMode = (): boolean => process.env.PEPR_MODE === "build"; -export const isDevMode = () => process.env.PEPR_MODE === "dev"; +export const isDevMode = (): boolean => process.env.PEPR_MODE === "dev"; export class PeprModule { #controller!: Controller; @@ -135,7 +135,7 @@ export class PeprModule { * * @param port */ - start = (port = 3000) => { + start = (port = 3000): void => { this.#controller.startServer(port); }; } From 34f4e50025cad966d2e78cc1c55aa17530e1d77f Mon Sep 17 00:00:00 2001 From: Sam Mayer Date: Fri, 13 Dec 2024 11:05:30 -0600 Subject: [PATCH 29/29] chore: reduce complexity of helpers.ts (#1575) ## Description This PR addresses complexity in `helpers.ts` End to End Test: (See [Pepr Excellent Examples](https://github.com/defenseunicorns/pepr-excellent-examples)) ## Related Issue Fixes #1534 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Unit, [Journey](https://github.com/defenseunicorns/pepr/tree/main/journey), [E2E Tests](https://github.com/defenseunicorns/pepr-excellent-examples), [docs](https://github.com/defenseunicorns/pepr/tree/main/docs), [adr](https://github.com/defenseunicorns/pepr/tree/main/adr) added or updated as needed - [x] [Contributor Guide Steps](https://docs.pepr.dev/main/contribute/#submitting-a-pull-request) followed --- src/lib/filter/filterNoMatchReason.ts | 108 +++++++++++++++++++ src/lib/helpers.test.ts | 36 ++----- src/lib/helpers.ts | 145 +++----------------------- src/lib/watch-processor.ts | 2 +- 4 files changed, 135 insertions(+), 156 deletions(-) create mode 100644 src/lib/filter/filterNoMatchReason.ts diff --git a/src/lib/filter/filterNoMatchReason.ts b/src/lib/filter/filterNoMatchReason.ts new file mode 100644 index 000000000..e9f64133d --- /dev/null +++ b/src/lib/filter/filterNoMatchReason.ts @@ -0,0 +1,108 @@ +import { KubernetesObject } from "kubernetes-fluent-client"; +import { + mismatchedDeletionTimestamp, + mismatchedName, + definedName, + carriedName, + misboundNamespace, + mismatchedLabels, + definedLabels, + carriedLabels, + mismatchedAnnotations, + definedAnnotations, + carriedAnnotations, + uncarryableNamespace, + carriedNamespace, + unbindableNamespaces, + definedNamespaces, + mismatchedNamespace, + mismatchedNamespaceRegex, + definedNamespaceRegexes, + mismatchedNameRegex, + definedNameRegex, + carriesIgnoredNamespace, + missingCarriableNamespace, +} from "./adjudicators/adjudicators"; +import { Binding } from "../types"; + +/** + * Decide to run callback after the event comes back from API Server + **/ + +export function filterNoMatchReason( + binding: Binding, + kubernetesObject: Partial, + capabilityNamespaces: string[], + ignoredNamespaces?: string[], +): string { + const prefix = "Ignoring Watch Callback:"; + + // prettier-ignore + return ( + mismatchedDeletionTimestamp(binding, kubernetesObject) ? + `${prefix} Binding defines deletionTimestamp but Object does not carry it.` : + + mismatchedName(binding, kubernetesObject) ? + `${prefix} Binding defines name '${definedName(binding)}' but Object carries '${carriedName(kubernetesObject)}'.` : + + misboundNamespace(binding) ? + `${prefix} Cannot use namespace filter on a namespace object.` : + + mismatchedLabels(binding, kubernetesObject) ? + ( + `${prefix} Binding defines labels '${JSON.stringify(definedLabels(binding))}' ` + + `but Object carries '${JSON.stringify(carriedLabels(kubernetesObject))}'.` + ) : + + mismatchedAnnotations(binding, kubernetesObject) ? + ( + `${prefix} Binding defines annotations '${JSON.stringify(definedAnnotations(binding))}' ` + + `but Object carries '${JSON.stringify(carriedAnnotations(kubernetesObject))}'.` + ) : + + uncarryableNamespace(capabilityNamespaces, kubernetesObject) ? + ( + `${prefix} Object carries namespace '${carriedNamespace(kubernetesObject)}' ` + + `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` + ) : + + unbindableNamespaces(capabilityNamespaces, binding) ? + ( + `${prefix} Binding defines namespaces ${JSON.stringify(definedNamespaces(binding))} ` + + `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` + ) : + + mismatchedNamespace(binding, kubernetesObject) ? + ( + `${prefix} Binding defines namespaces '${JSON.stringify(definedNamespaces(binding))}' ` + + `but Object carries '${carriedNamespace(kubernetesObject)}'.` + ) : + + mismatchedNamespaceRegex(binding, kubernetesObject) ? + ( + `${prefix} Binding defines namespace regexes ` + + `'${JSON.stringify(definedNamespaceRegexes(binding))}' ` + + `but Object carries '${carriedNamespace(kubernetesObject)}'.` + ) : + + mismatchedNameRegex(binding, kubernetesObject) ? + ( + `${prefix} Binding defines name regex '${definedNameRegex(binding)}' ` + + `but Object carries '${carriedName(kubernetesObject)}'.` + ) : + + carriesIgnoredNamespace(ignoredNamespaces, kubernetesObject) ? + ( + `${prefix} Object carries namespace '${carriedNamespace(kubernetesObject)}' ` + + `but ignored namespaces include '${JSON.stringify(ignoredNamespaces)}'.` + ) : + + missingCarriableNamespace(capabilityNamespaces, kubernetesObject) ? + ( + `${prefix} Object does not carry a namespace ` + + `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` + ) : + + "" + ); +} diff --git a/src/lib/helpers.test.ts b/src/lib/helpers.test.ts index 8fdfd5511..78ce1203a 100644 --- a/src/lib/helpers.test.ts +++ b/src/lib/helpers.test.ts @@ -4,10 +4,8 @@ import { Binding, CapabilityExport } from "./types"; import { Event } from "./enums"; import { - addVerbIfNotExists, bindingAndCapabilityNSConflict, createRBACMap, - filterNoMatchReason, dedent, generateWatchNamespaceError, hasAnyOverlap, @@ -22,6 +20,7 @@ import { validateCapabilityNames, ValidationError, } from "./helpers"; +import { filterNoMatchReason } from "./filter/filterNoMatchReason"; import { sanitizeResourceName } from "../sdk/sdk"; import * as fc from "fast-check"; import { expect, describe, jest, beforeEach, afterEach, it } from "@jest/globals"; @@ -358,20 +357,6 @@ describe("createRBACMap", () => { }); }); -describe("addVerbIfNotExists", () => { - it("should add a verb if it does not exist in the array", () => { - const verbs = ["get", "list"]; - addVerbIfNotExists(verbs, "watch"); - expect(verbs).toEqual(["get", "list", "watch"]); - }); - - it("should not add a verb if it already exists in the array", () => { - const verbs = ["get", "list", "watch"]; - addVerbIfNotExists(verbs, "get"); - expect(verbs).toEqual(["get", "list", "watch"]); // The array remains unchanged - }); -}); - describe("hasAnyOverlap", () => { it("returns true for overlapping arrays", () => { expect(hasAnyOverlap([1, 2, 3], [3, 4, 5])).toBe(true); @@ -683,26 +668,25 @@ describe("namespaceComplianceValidator", () => { }); describe("parseTimeout", () => { - const PREV = "a"; it("should return a number when a valid string number between 1 and 30 is provided", () => { - expect(parseTimeout("5", PREV)).toBe(5); - expect(parseTimeout("1", PREV)).toBe(1); - expect(parseTimeout("30", PREV)).toBe(30); + expect(parseTimeout("5")).toBe(5); + expect(parseTimeout("1")).toBe(1); + expect(parseTimeout("30")).toBe(30); }); it("should throw an InvalidArgumentError for non-numeric strings", () => { - expect(() => parseTimeout("abc", PREV)).toThrow(Error); - expect(() => parseTimeout("", PREV)).toThrow(Error); + expect(() => parseTimeout("abc")).toThrow(Error); + expect(() => parseTimeout("")).toThrow(Error); }); it("should throw an InvalidArgumentError for numbers outside the 1-30 range", () => { - expect(() => parseTimeout("0", PREV)).toThrow(Error); - expect(() => parseTimeout("31", PREV)).toThrow(Error); + expect(() => parseTimeout("0")).toThrow(Error); + expect(() => parseTimeout("31")).toThrow(Error); }); it("should throw an InvalidArgumentError for numeric strings that represent floating point numbers", () => { - expect(() => parseTimeout("5.5", PREV)).toThrow(Error); - expect(() => parseTimeout("20.1", PREV)).toThrow(Error); + expect(() => parseTimeout("5.5")).toThrow(Error); + expect(() => parseTimeout("20.1")).toThrow(Error); }); }); diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index e36663cd2..897b616a9 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -1,34 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors -import { KubernetesObject } from "kubernetes-fluent-client"; import Log from "./telemetry/logger"; import { Binding, CapabilityExport } from "./types"; import { sanitizeResourceName } from "../sdk/sdk"; -import { - carriedAnnotations, - carriedLabels, - carriedName, - carriedNamespace, - carriesIgnoredNamespace, - definedAnnotations, - definedLabels, - definedName, - definedNameRegex, - definedNamespaces, - definedNamespaceRegexes, - misboundNamespace, - mismatchedAnnotations, - mismatchedDeletionTimestamp, - mismatchedLabels, - mismatchedName, - mismatchedNameRegex, - mismatchedNamespace, - mismatchedNamespaceRegex, - missingCarriableNamespace, - unbindableNamespaces, - uncarryableNamespace, -} from "./filter/adjudicators/adjudicators"; export function matchesRegex(pattern: string, testString: string): boolean { return new RegExp(pattern).test(testString); @@ -55,100 +30,13 @@ export function validateHash(expectedHash: string): void { } } -export type RBACMap = { +type RBACMap = { [key: string]: { verbs: string[]; plural: string; }; }; -/** - * Decide to run callback after the event comes back from API Server - **/ -export function filterNoMatchReason( - binding: Binding, - kubernetesObject: Partial, - capabilityNamespaces: string[], - ignoredNamespaces?: string[], -): string { - const prefix = "Ignoring Watch Callback:"; - - // prettier-ignore - return ( - mismatchedDeletionTimestamp(binding, kubernetesObject) ? - `${prefix} Binding defines deletionTimestamp but Object does not carry it.` : - - mismatchedName(binding, kubernetesObject) ? - `${prefix} Binding defines name '${definedName(binding)}' but Object carries '${carriedName(kubernetesObject)}'.` : - - misboundNamespace(binding) ? - `${prefix} Cannot use namespace filter on a namespace object.` : - - mismatchedLabels(binding, kubernetesObject) ? - ( - `${prefix} Binding defines labels '${JSON.stringify(definedLabels(binding))}' ` + - `but Object carries '${JSON.stringify(carriedLabels(kubernetesObject))}'.` - ) : - - mismatchedAnnotations(binding, kubernetesObject) ? - ( - `${prefix} Binding defines annotations '${JSON.stringify(definedAnnotations(binding))}' ` + - `but Object carries '${JSON.stringify(carriedAnnotations(kubernetesObject))}'.` - ) : - - uncarryableNamespace(capabilityNamespaces, kubernetesObject) ? - ( - `${prefix} Object carries namespace '${carriedNamespace(kubernetesObject)}' ` + - `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` - ) : - - unbindableNamespaces(capabilityNamespaces, binding) ? - ( - `${prefix} Binding defines namespaces ${JSON.stringify(definedNamespaces(binding))} ` + - `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` - ) : - - mismatchedNamespace(binding, kubernetesObject) ? - ( - `${prefix} Binding defines namespaces '${JSON.stringify(definedNamespaces(binding))}' ` + - `but Object carries '${carriedNamespace(kubernetesObject)}'.` - ) : - - mismatchedNamespaceRegex(binding, kubernetesObject) ? - ( - `${prefix} Binding defines namespace regexes ` + - `'${JSON.stringify(definedNamespaceRegexes(binding))}' ` + - `but Object carries '${carriedNamespace(kubernetesObject)}'.` - ) : - - mismatchedNameRegex(binding, kubernetesObject) ? - ( - `${prefix} Binding defines name regex '${definedNameRegex(binding)}' ` + - `but Object carries '${carriedName(kubernetesObject)}'.` - ) : - - carriesIgnoredNamespace(ignoredNamespaces, kubernetesObject) ? - ( - `${prefix} Object carries namespace '${carriedNamespace(kubernetesObject)}' ` + - `but ignored namespaces include '${JSON.stringify(ignoredNamespaces)}'.` - ) : - - missingCarriableNamespace(capabilityNamespaces, kubernetesObject) ? - ( - `${prefix} Object does not carry a namespace ` + - `but namespaces allowed by Capability are '${JSON.stringify(capabilityNamespaces)}'.` - ) : - - "" - ); -} - -export function addVerbIfNotExists(verbs: string[], verb: string): void { - if (!verbs.includes(verb)) { - verbs.push(verb); - } -} - export function createRBACMap(capabilities: CapabilityExport[]): RBACMap { return capabilities.reduce((acc: RBACMap, capability: CapabilityExport) => { capability.bindings.forEach(binding => { @@ -256,13 +144,16 @@ export function namespaceComplianceValidator(capability: CapabilityExport, ignor } // Ensure that each regexNamespace matches a capabilityNamespace + matchRegexToCapababilityNamespace(bindingRegexNamespaces, capabilityNamespaces); + // ensure regexNamespaces do not match ignored ns + checkRegexNamespaces(bindingRegexNamespaces, ignoredNamespaces); +} - if ( - bindingRegexNamespaces && - bindingRegexNamespaces.length > 0 && - capabilityNamespaces && - capabilityNamespaces.length > 0 - ) { +const matchRegexToCapababilityNamespace = ( + bindingRegexNamespaces: string[], + capabilityNamespaces: string[] | undefined, +): void => { + if (bindingRegexNamespaces.length > 0 && capabilityNamespaces && capabilityNamespaces.length > 0) { for (const regexNamespace of bindingRegexNamespaces) { let matches = false; matches = @@ -275,13 +166,10 @@ export function namespaceComplianceValidator(capability: CapabilityExport, ignor } } } - // ensure regexNamespaces do not match ignored ns - if ( - bindingRegexNamespaces && - bindingRegexNamespaces.length > 0 && - ignoredNamespaces && - ignoredNamespaces.length > 0 - ) { +}; + +const checkRegexNamespaces = (bindingRegexNamespaces: string[], ignoredNamespaces: string[] | undefined): void => { + if (bindingRegexNamespaces.length > 0 && ignoredNamespaces && ignoredNamespaces.length > 0) { for (const regexNamespace of bindingRegexNamespaces) { const matchedNS = ignoredNamespaces.find(ignoredNS => matchesRegex(regexNamespace, ignoredNS)); if (matchedNS) { @@ -291,7 +179,7 @@ export function namespaceComplianceValidator(capability: CapabilityExport, ignor } } } -} +}; // check if secret is over the size limit export function secretOverLimit(str: string): boolean { @@ -302,8 +190,7 @@ export function secretOverLimit(str: string): boolean { return sizeInBytes > oneMiBInBytes; } -/* eslint-disable @typescript-eslint/no-unused-vars */ -export const parseTimeout = (value: string, previous: unknown): number => { +export const parseTimeout = (value: string): number => { const parsedValue = parseInt(value, 10); const floatValue = parseFloat(value); if (isNaN(parsedValue)) { diff --git a/src/lib/watch-processor.ts b/src/lib/watch-processor.ts index 1f36a9c9c..8a759b37a 100644 --- a/src/lib/watch-processor.ts +++ b/src/lib/watch-processor.ts @@ -3,7 +3,7 @@ import { K8s, KubernetesObject, WatchCfg, WatchEvent } from "kubernetes-fluent-client"; import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types"; import { Capability } from "./capability"; -import { filterNoMatchReason } from "./helpers"; +import { filterNoMatchReason } from "./filter/filterNoMatchReason"; import { removeFinalizer } from "./finalizer"; import Log from "./telemetry/logger"; import { Queue } from "./queue";