diff --git a/package.json b/package.json index 00dcf69e1..3c0cdbd29 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uds-core", - "version": "0.4.0", + "version": "0.5.0", "description": "A collection of capabilities for UDS Core", "keywords": [ "pepr", diff --git a/pepr.ts b/pepr.ts index 3e9e3ca31..da8592ef6 100644 --- a/pepr.ts +++ b/pepr.ts @@ -1,22 +1,42 @@ -import { PeprModule } from "pepr"; +import { Log, PeprModule } from "pepr"; import cfg from "./package.json"; import { istio } from "./src/pepr/istio"; import { operator } from "./src/pepr/operator"; -import { policies } from "./src/pepr/policies"; +import { Policy } from "./src/pepr/operator/crd"; +import { registerCRDs } from "./src/pepr/operator/crd/register"; +import { policies, startExemptionWatch } from "./src/pepr/policies"; import { prometheus } from "./src/pepr/prometheus"; -new PeprModule(cfg, [ - // UDS Core Operator - operator, +(async () => { + // Apply the CRDs to the cluster + await registerCRDs(); + // KFC watch for exemptions and update in-memory map + await startExemptionWatch(); + new PeprModule(cfg, [ + // UDS Core Operator + operator, - // UDS Core Policies - policies, + // UDS Core Policies + policies, - // Istio service mesh - istio, + // Istio service mesh + istio, - // Prometheus monitoring stack - prometheus, -]); + // Prometheus monitoring stack + prometheus, + ]); + // Remove legacy policy entries from the pepr store for the 0.5.0 upgrade + if (process.env.PEPR_WATCH_MODE === "true" && cfg.version === "0.5.0") { + Log.debug("Clearing legacy pepr store exemption entries..."); + policies.Store.onReady(() => { + for (const p of Object.values(Policy)) { + policies.Store.removeItem(p); + } + }); + } +})().catch(err => { + Log.error(err); + process.exit(1); +}); diff --git a/src/pepr/operator/common.ts b/src/pepr/operator/common.ts index 5c8e92c36..820e9d5f3 100644 --- a/src/pepr/operator/common.ts +++ b/src/pepr/operator/common.ts @@ -1,8 +1,5 @@ import { Capability } from "pepr"; -// Register the CRD -import "./crd/register"; - export const operator = new Capability({ name: "uds-core-operator", description: "The UDS Operator is responsible for managing the lifecycle of UDS resources", diff --git a/src/pepr/operator/controllers/exemptions/exemption-store.spec.ts b/src/pepr/operator/controllers/exemptions/exemption-store.spec.ts new file mode 100644 index 000000000..15e25d948 --- /dev/null +++ b/src/pepr/operator/controllers/exemptions/exemption-store.spec.ts @@ -0,0 +1,99 @@ +import { beforeEach, describe, expect, it } from "@jest/globals"; +import { Matcher, MatcherKind, Policy } from "../../crd"; +import { ExemptionStore } from "./exemption-store"; + +const enforcerMatcher = { + namespace: "neuvector", + name: "^neuvector-enforcer-pod.*", + kind: MatcherKind.Pod, +}; + +const controllerMatcher = { + namespace: "neuvector", + name: "^neuvector-controller-pod.*", + kind: MatcherKind.Pod, +}; + +const getExemption = (uid: string, matcher: Matcher, policies: Policy[]) => { + return { + metadata: { + uid, + }, + spec: { + exemptions: [ + { + matcher, + policies, + }, + ], + }, + }; +}; + +describe("Exemption Store", () => { + beforeEach(() => { + ExemptionStore.init(); + }); + + it("Add exemption", async () => { + const e = getExemption("uid", enforcerMatcher, [Policy.DisallowPrivileged]); + ExemptionStore.add(e); + const matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + + expect(matchers).toHaveLength(1); + }); + + it("Delete exemption", async () => { + const e = getExemption("uid", enforcerMatcher, [Policy.DisallowPrivileged]); + ExemptionStore.add(e); + let matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(1); + ExemptionStore.remove(e); + + matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(0); + }); + + it("Update exemption", async () => { + const enforcerException = getExemption("uid", enforcerMatcher, [Policy.DisallowPrivileged]); + ExemptionStore.add(enforcerException); + + let matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(1); + + const controllerExemption = getExemption("uid", controllerMatcher, [Policy.RequireNonRootUser]); + ExemptionStore.add(controllerExemption); + + matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(0); + }); + + it("Add multiple policies", async () => { + const enforcerException = getExemption("foo", enforcerMatcher, [Policy.DisallowPrivileged]); + ExemptionStore.add(enforcerException); + + let matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(1); + + const controllerExemption = getExemption("bar", controllerMatcher, [Policy.RequireNonRootUser]); + ExemptionStore.add(controllerExemption); + + matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(1); + + matchers = ExemptionStore.getByPolicy(Policy.RequireNonRootUser); + expect(matchers).toHaveLength(1); + }); + + it("Add duplicate exemptions owned by different owners", async () => { + const enforcerException = getExemption("foo", enforcerMatcher, [Policy.DisallowPrivileged]); + const otherEnforcerException = getExemption("bar", enforcerMatcher, [ + Policy.DisallowPrivileged, + ]); + ExemptionStore.add(enforcerException); + ExemptionStore.add(otherEnforcerException); + + const matchers = ExemptionStore.getByPolicy(Policy.DisallowPrivileged); + expect(matchers).toHaveLength(2); + }); +}); diff --git a/src/pepr/operator/controllers/exemptions/exemption-store.ts b/src/pepr/operator/controllers/exemptions/exemption-store.ts new file mode 100644 index 000000000..3cb024de8 --- /dev/null +++ b/src/pepr/operator/controllers/exemptions/exemption-store.ts @@ -0,0 +1,83 @@ +import { Log } from "pepr"; +import { StoredMatcher } from "../../../policies"; +import { Matcher, Policy, UDSExemption } from "../../crd"; + +export type PolicyOwnerMap = Map; +export type PolicyMap = Map; +let policyExemptionMap: PolicyMap; +let policyOwnerMap: PolicyOwnerMap; + +function init(): void { + policyExemptionMap = new Map(); + policyOwnerMap = new Map(); + for (const p of Object.values(Policy)) { + policyExemptionMap.set(p, []); + } +} + +function getByPolicy(policy: Policy): StoredMatcher[] { + return policyExemptionMap.get(policy) || []; +} + +function setByPolicy(policy: Policy, matchers: StoredMatcher[]): void { + policyExemptionMap.set(policy, matchers); +} + +function addMatcher(matcher: Matcher, p: Policy, owner: string = ""): void { + const storedMatcher = { + ...matcher, + owner, + }; + + const storedMatchers = getByPolicy(p); + storedMatchers.push(storedMatcher); +} + +// Iterate through each exemption block of CR and add matchers to PolicyMap +function add(exemption: UDSExemption, log: boolean = true) { + // Remove any existing exemption for this owner, in case of WatchPhase.Modified + remove(exemption); + const owner = exemption.metadata?.uid || ""; + policyOwnerMap.set(owner, exemption); + + for (const e of exemption.spec?.exemptions ?? []) { + const policies = e.policies ?? []; + for (const p of policies) { + // Append the matcher to the list of stored matchers for this policy + addMatcher(e.matcher, p, owner); + if (log) { + Log.debug(`Added exemption to ${p}: ${JSON.stringify(e.matcher)}`); + } + } + } +} + +function remove(exemption: UDSExemption) { + const owner = exemption.metadata?.uid || ""; + const prevExemption = policyOwnerMap.get(owner); + + if (prevExemption) { + for (const e of prevExemption.spec?.exemptions ?? []) { + const policies = e.policies ?? []; + for (const p of policies) { + const existingMatchers = getByPolicy(p); + const filteredList = existingMatchers.filter(m => { + return m.owner !== owner; + }); + setByPolicy(p, filteredList); + } + } + policyOwnerMap.delete(owner); + Log.debug(`Removed all policy exemptions for ${owner}`); + } else { + Log.debug(`No existing exemption for owner ${owner}`); + } +} + +// export object with all included export as properties +export const ExemptionStore = { + init, + add, + remove, + getByPolicy, +}; diff --git a/src/pepr/operator/controllers/exemptions/exemptions.spec.ts b/src/pepr/operator/controllers/exemptions/exemptions.spec.ts index e48bccd68..8c276d879 100644 --- a/src/pepr/operator/controllers/exemptions/exemptions.spec.ts +++ b/src/pepr/operator/controllers/exemptions/exemptions.spec.ts @@ -1,10 +1,9 @@ -import { afterEach, beforeEach, describe, expect, it, jest } from "@jest/globals"; -import { Store } from "../../../policies/common"; +import { beforeEach, describe, expect, it } from "@jest/globals"; import { MatcherKind, Policy } from "../../crd"; import { Exemption } from "../../crd/generated/exemption-v1alpha1"; -import { processExemptions, removeExemptions } from "./exemptions"; +import { ExemptionStore } from "./exemption-store"; +import { WatchPhase, processExemptions } from "./exemptions"; -const mockStore = new Map(); const enforcerMatcher = { namespace: "neuvector", name: "^neuvector-enforcer-pod.*", @@ -53,141 +52,89 @@ const neuvectorMockExemption = { }, } as Exemption; -describe("Test processExemptions()", () => { +describe("Test processExemptions() no duplicate matchers in same CR", () => { beforeEach(() => { - jest.spyOn(Store, "getItem").mockImplementation((key: string) => { - return mockStore.get(key) || null; - }); - - jest.spyOn(Store, "setItem").mockImplementation((key: string, val: string) => { - mockStore.set(key, val); - }); - }); - - afterEach(() => { - mockStore.clear(); - jest.restoreAllMocks(); + ExemptionStore.init(); }); it("Add exemptions for the first time", async () => { - processExemptions(neuvectorMockExemption); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify(storedControllerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify( - storedControllerMatcher, - )},${JSON.stringify(storedPrometheusMatcher)}]`, - ); - }); - - it("Does not add duplicate matchers for same CR", async () => { - processExemptions(neuvectorMockExemption); - processExemptions(neuvectorMockExemption); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify(storedControllerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify( - storedControllerMatcher, - )},${JSON.stringify(storedPrometheusMatcher)}]`, - ); + processExemptions(neuvectorMockExemption, WatchPhase.Added); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + storedPrometheusMatcher, + ]); }); - it("Adds duplicate matchers if from separate CR", () => { - processExemptions(neuvectorMockExemption); - processExemptions({ ...neuvectorMockExemption, metadata: { uid: exemption2UID } }); - - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify( - storedControllerMatcher, - )},${JSON.stringify({ ...storedEnforcerMatcher, owner: exemption2UID })},${JSON.stringify({ - ...storedControllerMatcher, - owner: exemption2UID, - })}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify( - storedControllerMatcher, - )},${JSON.stringify(storedPrometheusMatcher)},${JSON.stringify({ - ...storedEnforcerMatcher, - owner: exemption2UID, - })},${JSON.stringify({ ...storedControllerMatcher, owner: exemption2UID })},${JSON.stringify({ - ...storedPrometheusMatcher, - owner: exemption2UID, - })}]`, - ); + it("Does not re-add matchers on updates", async () => { + processExemptions(neuvectorMockExemption, WatchPhase.Added); + processExemptions(neuvectorMockExemption, WatchPhase.Modified); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + storedPrometheusMatcher, + ]); }); - it("Removes exemptions from policy if policies removed from matcher policy list on update", async () => { + it("Handles updates -- remove policy, remove matcher, add policy, add matcher", async () => { + // remove RequireNonRootUser from enforcerMatcher + // remove prometheusMatcher + // add DisallowHostNamespaces to controllerMatcher + // add promtailMatcher with RequireNonRootUser const updatedNeuvectorExemption = { metadata: { uid: exemption1UID, }, spec: { exemptions: [ - { matcher: enforcerMatcher, policies: [Policy.DisallowPrivileged] }, { - matcher: controllerMatcher, - policies: [Policy.DropAllCapabilities], - }, - { - matcher: prometheusMatcher, - policies: [Policy.DropAllCapabilities], + matcher: enforcerMatcher, + policies: [Policy.DisallowPrivileged, Policy.DropAllCapabilities], }, - ], - }, - } as Exemption; - - processExemptions(neuvectorMockExemption); - processExemptions(updatedNeuvectorExemption); - - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual("[]"); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedControllerMatcher)},${JSON.stringify(storedPrometheusMatcher)}]`, - ); - }); - - it("Removes matchers from policy if matchers removed from CR", () => { - const updatedNeuvectorExemption = { - metadata: { - uid: exemption1UID, - }, - spec: { - exemptions: [ { matcher: controllerMatcher, - policies: [Policy.DisallowPrivileged, Policy.DropAllCapabilities], + policies: [ + Policy.DisallowPrivileged, + Policy.DropAllCapabilities, + Policy.DisallowHostNamespaces, + ], }, { - matcher: { ...enforcerMatcher, kind: MatcherKind.Service }, - policies: [Policy.DisallowNodePortServices], + matcher: promtailMatcher, + policies: [Policy.RequireNonRootUser], }, ], }, } as Exemption; - processExemptions(neuvectorMockExemption); - processExemptions(updatedNeuvectorExemption); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual("[]"); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedControllerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedControllerMatcher)}]`, - ); - expect(Store.getItem(Policy.DisallowNodePortServices)).toEqual( - `[${JSON.stringify({ ...storedEnforcerMatcher, kind: MatcherKind.Service })}]`, - ); + processExemptions(neuvectorMockExemption, WatchPhase.Added); + processExemptions(updatedNeuvectorExemption, WatchPhase.Modified); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([ + { ...storedPromtailMatcher, owner: exemption1UID }, + ]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DisallowHostNamespaces)).toEqual([ + storedControllerMatcher, + ]); }); - it("Adds duplicate exemptions set by same CR if different matcher kind", () => { + it("Adds duplicate exemptions set by same CR if different matcher kind", async () => { const neuvectorMockExemption2 = { metadata: { uid: exemption1UID, @@ -210,23 +157,17 @@ describe("Test processExemptions()", () => { }, } as Exemption; - processExemptions(neuvectorMockExemption2); - - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.DisallowNodePortServices)).toEqual( - `[${JSON.stringify({ ...storedEnforcerMatcher, kind: MatcherKind.Service })}]`, - ); + processExemptions(neuvectorMockExemption2, WatchPhase.Added); + + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowNodePortServices)).toEqual([ + { ...storedEnforcerMatcher, kind: MatcherKind.Service }, + ]); }); - it("Adds duplicate exemptions set by same CR if different namespace", () => { + it("Adds duplicate exemptions set by same CR if different namespace", async () => { const diffNS = "differentNS"; const neuvectorMockExemption2 = { metadata: { @@ -254,29 +195,32 @@ describe("Test processExemptions()", () => { }, } as Exemption; - processExemptions(neuvectorMockExemption2); + processExemptions(neuvectorMockExemption2, WatchPhase.Added); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify({ + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + { ...storedEnforcerMatcher, namespace: diffNS, - })}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify({ + }, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([ + storedEnforcerMatcher, + { ...storedEnforcerMatcher, namespace: diffNS, - })}]`, - ); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify({ + }, + ]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([ + storedEnforcerMatcher, + { ...storedEnforcerMatcher, namespace: diffNS, - })}]`, - ); + }, + ]); }); - it("Adds duplicate exemptions set by same CR if different namespace and different policy list", () => { + it("Adds duplicate exemptions set by same CR if different namespace and different policy list", async () => { const diffNS = "differentNS"; const neuvectorMockExemption2 = { metadata: { @@ -300,47 +244,121 @@ describe("Test processExemptions()", () => { }, } as Exemption; - processExemptions(neuvectorMockExemption2); + processExemptions(neuvectorMockExemption2, WatchPhase.Added); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)},${JSON.stringify({ + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + { ...storedEnforcerMatcher, namespace: diffNS, - })}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); + }, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); }); }); -describe("Test removeExemptions()", () => { +describe("Test processExemptions() duplicate matchers in same CR", () => { beforeEach(() => { - jest.spyOn(Store, "getItem").mockImplementation((key: string) => { - return mockStore.get(key) || null; - }); + ExemptionStore.init(); + }); + + const sameMatcherMockExemption = { + metadata: { + uid: exemption1UID, + }, + spec: { + exemptions: [ + { + matcher: enforcerMatcher, + policies: [Policy.DisallowPrivileged], + }, + { + matcher: enforcerMatcher, + policies: [Policy.RequireNonRootUser], + }, + { + matcher: enforcerMatcher, + policies: [Policy.DropAllCapabilities], + }, + ], + }, + }; + + it("Adds same matchers with different policies", () => { + processExemptions(sameMatcherMockExemption, WatchPhase.Added); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + }); + + it("Does not re-add matchers on updates", () => { + processExemptions(sameMatcherMockExemption, WatchPhase.Added); + processExemptions(sameMatcherMockExemption, WatchPhase.Modified); + + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + }); + + it.only("Handles updates - remove policy, remove matcher, add policy, add matcher", async () => { + // remove RequireNonRoot from enforcerMatcher (satisfies remove matcher in this duplicate case) + // add DisallowHostNamespaces to enforcerMatcher + // add controllerMatcher with DisallowPrivileged + const updateSameMatcherMock = { + metadata: { + uid: exemption1UID, + }, + spec: { + exemptions: [ + { + matcher: enforcerMatcher, + policies: [Policy.DisallowPrivileged], + }, + { + matcher: enforcerMatcher, + policies: [Policy.DropAllCapabilities], + }, + { + matcher: enforcerMatcher, + policies: [Policy.DisallowHostNamespaces], + }, + { + matcher: controllerMatcher, + policies: [Policy.DisallowPrivileged], + }, + ], + }, + } as Exemption; - jest.spyOn(Store, "setItem").mockImplementation((key: string, val: string) => { - mockStore.set(key, val); - }); + processExemptions(sameMatcherMockExemption, WatchPhase.Added); + processExemptions(updateSameMatcherMock, WatchPhase.Modified); + + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ + storedEnforcerMatcher, + storedControllerMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowHostNamespaces)).toEqual([ + storedEnforcerMatcher, + ]); }); +}); - afterEach(() => { - mockStore.clear(); - jest.restoreAllMocks(); +describe("Test processExemptions(); phase DELETED", () => { + beforeEach(() => { + ExemptionStore.init(); }); - it("Removes all CRs exemptions when deleted", () => { - processExemptions(neuvectorMockExemption); - removeExemptions(neuvectorMockExemption); - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual("[]"); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual("[]"); + it("Removes all CRs exemptions when deleted", async () => { + processExemptions(neuvectorMockExemption, WatchPhase.Added); + processExemptions(neuvectorMockExemption, WatchPhase.Deleted); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([]); }); - it("Does not remove exemptions set by separate CR from the one being deleted", () => { + it("Does not remove exemptions set by separate CR from the one being deleted", async () => { const promtailMockExemption = { metadata: { uid: exemption2UID, @@ -359,22 +377,16 @@ describe("Test removeExemptions()", () => { }, } as Exemption; - processExemptions(neuvectorMockExemption); - processExemptions(promtailMockExemption); - removeExemptions(neuvectorMockExemption); - - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedPromtailMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedPromtailMatcher)}]`, - ); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedPromtailMatcher)}]`, - ); + processExemptions(neuvectorMockExemption, WatchPhase.Added); + processExemptions(promtailMockExemption, WatchPhase.Added); + processExemptions(neuvectorMockExemption, WatchPhase.Deleted); + + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedPromtailMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedPromtailMatcher]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedPromtailMatcher]); }); - it("Does not delete duplicate exemptions if set by separate CRs", () => { + it("Does not delete duplicate exemptions if set by separate CRs", async () => { const neuvectorMockExemption2 = { metadata: { uid: exemption1UID, @@ -393,7 +405,7 @@ describe("Test removeExemptions()", () => { }, } as Exemption; - const neuvectorDuplicatMockExemption = { + const neuvectorDuplicateMockExemption = { metadata: { uid: exemption2UID, }, @@ -411,18 +423,67 @@ describe("Test removeExemptions()", () => { }, } as Exemption; - processExemptions(neuvectorMockExemption2); - processExemptions(neuvectorDuplicatMockExemption); - removeExemptions(neuvectorDuplicatMockExemption); - - expect(Store.getItem(Policy.DisallowPrivileged)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.DropAllCapabilities)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); - expect(Store.getItem(Policy.RequireNonRootUser)).toEqual( - `[${JSON.stringify(storedEnforcerMatcher)}]`, - ); + processExemptions(neuvectorMockExemption2, WatchPhase.Added); + processExemptions(neuvectorDuplicateMockExemption, WatchPhase.Added); + processExemptions(neuvectorDuplicateMockExemption, WatchPhase.Deleted); + + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedEnforcerMatcher]); + }); + + it("Does not delete exemptions for the same policies from separate CRs during modification", async () => { + const neuvectorMockExemption = { + metadata: { + uid: exemption1UID, + }, + spec: { + exemptions: [ + { + matcher: enforcerMatcher, + policies: [Policy.RequireNonRootUser, Policy.DropAllCapabilities], + }, + ], + }, + } as Exemption; + + const promtailMockExemption = { + metadata: { + uid: exemption2UID, + }, + spec: { + exemptions: [ + { + matcher: promtailMatcher, + policies: [Policy.DisallowPrivileged], + }, + ], + }, + } as Exemption; + + const promtailUpdatedMockExemption = { + metadata: { + uid: exemption2UID, + }, + spec: { + exemptions: [ + { + matcher: promtailMatcher, + policies: [Policy.DisallowPrivileged, Policy.RequireNonRootUser], + }, + ], + }, + } as Exemption; + + processExemptions(neuvectorMockExemption, WatchPhase.Added); + processExemptions(promtailMockExemption, WatchPhase.Added); + processExemptions(promtailUpdatedMockExemption, WatchPhase.Modified); + + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([ + storedEnforcerMatcher, + storedPromtailMatcher, + ]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedPromtailMatcher]); }); }); diff --git a/src/pepr/operator/controllers/exemptions/exemptions.ts b/src/pepr/operator/controllers/exemptions/exemptions.ts index 884633e56..a52a39723 100644 --- a/src/pepr/operator/controllers/exemptions/exemptions.ts +++ b/src/pepr/operator/controllers/exemptions/exemptions.ts @@ -1,160 +1,24 @@ -import { Log } from "pepr"; -import { policies } from "../../../policies/index"; -import { ExemptionElement, Matcher, Policy, UDSExemption } from "../../crd"; - -type StoredMatcher = Matcher & { owner: string }; -type PolicyMap = Map; - -const isSame = (a: StoredMatcher, b: StoredMatcher) => { - return ( - a.name === b.name && a.namespace === b.namespace && a.kind == b.kind && a.owner === b.owner - ); -}; - -function addIfIncludesPolicy( - policy: Policy, - policyMap: PolicyMap, - exemptionEl: ExemptionElement, - ownerId: string, -) { - const storedMatchers = policyMap.get(policy) ?? []; - const matcherToStore = { - ...exemptionEl.matcher, - owner: ownerId, - }; - const isDuplicate = storedMatchers.some(sm => isSame(sm, matcherToStore)); - - // add if not already added to policy's exemption list - if (exemptionEl.policies.includes(policy) && !isDuplicate) { - policyMap.set(policy, [...storedMatchers, matcherToStore]); - } -} - -// Delete a matcher from a policy if the policy has been removed from its policy list -function deleteIfPolicyRemoved( - policy: Policy, - policyMap: PolicyMap, - exemptionEl: ExemptionElement, - ownerId: string, -) { - const matcher = { - ...exemptionEl.matcher, - owner: ownerId, - }; - const storedMatchers = policyMap.get(policy) ?? []; - - if (storedMatchers.some(sm => isSame(sm, matcher)) && !exemptionEl.policies.includes(policy)) { - policyMap.set( - policy, - storedMatchers.filter(sm => { - if (!isSame(sm, matcher)) return sm; - }), - ); - Log.debug(`Removing ${matcher.name} from ${policy}`); - } +import { UDSExemption } from "../../crd"; +import { ExemptionStore } from "./exemption-store"; + +export enum WatchPhase { + Added = "ADDED", + Modified = "MODIFIED", + Deleted = "DELETED", + Bookmark = "BOOKMARK", + Error = "ERROR", } -// Delete matchers from the store if they no longer exist on a UDSExemption -function deleteIfMatchersRemoved( - policy: Policy, - policyMap: PolicyMap, - currExemptMatchers: StoredMatcher[], - ownerId: string, -) { - const policyMatchers = policyMap.get(policy) || []; - - // Check stored matchers that have same owner ref as current UDSExemption - for (const pm of policyMatchers.filter(m => m.owner === ownerId)) { - let shouldBeRemoved = true; - - // check if stored matcher exists in current list of UDSExemption matchers - for (const m of currExemptMatchers) { - if (isSame(pm, m)) { - shouldBeRemoved = false; - } - } - - if (shouldBeRemoved) { - // get again incase matchers were updated on previous iteration - const updatedPolicyMatchers = policyMap.get(policy) || []; - policyMap.set( - policy, - updatedPolicyMatchers.filter(sm => { - if (!isSame(sm, pm)) return sm; - }), - ); - - Log.debug(`Removing ${pm.name} from ${policy}`); - } +// Handle adding, updating, and deleting exemptions from Policymap +export function processExemptions(exemption: UDSExemption, phase: WatchPhase) { + switch (phase) { + case WatchPhase.Added: + case WatchPhase.Modified: + ExemptionStore.add(exemption); + break; + + case WatchPhase.Deleted: + ExemptionStore.remove(exemption); + break; } } - -// Iterate through local Map and update Store -function updateStore(policyMap: PolicyMap) { - const { Store } = policies; - for (const [policy, matchers] of policyMap.entries()) { - Log.debug(`Updating uds policy ${policy} exemptions: ${JSON.stringify(matchers)}`); - Store.setItem(policy, JSON.stringify(matchers)); - } -} - -function setupPolicyMap() { - const { Store } = policies; - const policyMap: PolicyMap = new Map(); - const policyList = Object.values(Policy); - - for (const p of policyList) { - policyMap.set(p, JSON.parse(Store.getItem(p) || "[]")); - } - - return { policyMap, policyList }; -} - -// Add Exemptions to Pepr store as "policy": "[{...matcher, owner: uid}]" -//(Performance Optimization) Use local map to do aggregation before updating store -export function processExemptions(exempt: UDSExemption) { - const { policyMap, policyList } = setupPolicyMap(); - const currExemptMatchers: StoredMatcher[] = []; - const ownerId = exempt.metadata?.uid || ""; - - // Iterate through all policies -- important for removing exemptions if CR is updated - for (const p of policyList) { - for (const e of exempt.spec?.exemptions ?? []) { - currExemptMatchers.push({ - ...e.matcher, - owner: ownerId, - }); - - // Add if exemption has this policy in its list - addIfIncludesPolicy(p, policyMap, e, ownerId); - - // Check if matcher no longer has this policy from previous CR version - deleteIfPolicyRemoved(p, policyMap, e, ownerId); - } - - // Check if policy should no longer have this matcher from previous CR version - deleteIfMatchersRemoved(p, policyMap, currExemptMatchers, ownerId); - } - - updateStore(policyMap); -} - -//(Performance Optimization) Use local map to do aggregation before updating store -export function removeExemptions(exempt: UDSExemption) { - const { policyMap } = setupPolicyMap(); - - Log.debug(`Removing policy exemptions for ${exempt.metadata?.name}`); - - // Loop through exemptions and remove matchers from policies in the local map - for (const e of exempt.spec?.exemptions ?? []) { - for (const p of e.policies) { - const matchers = policyMap.get(p) || []; - const filteredList = matchers.filter(m => { - if (!isSame(m, { ...e.matcher, owner: exempt.metadata?.uid || "" })) return m; - }); - policyMap.set(p, filteredList); - } - } - - updateStore(policyMap); -} diff --git a/src/pepr/operator/crd/generated/exemption-v1alpha1.ts b/src/pepr/operator/crd/generated/exemption-v1alpha1.ts index 6f4347151..487c4961f 100644 --- a/src/pepr/operator/crd/generated/exemption-v1alpha1.ts +++ b/src/pepr/operator/crd/generated/exemption-v1alpha1.ts @@ -4,14 +4,13 @@ import { GenericKind, RegisterKind } from "kubernetes-fluent-client"; export class Exemption extends GenericKind { spec?: Spec; - status?: Status; } export interface Spec { /** * Policy exemptions */ - exemptions?: ExemptionElement[]; + exemptions: ExemptionElement[]; } export interface ExemptionElement { @@ -64,19 +63,6 @@ export enum Policy { RestrictVolumeTypes = "RestrictVolumeTypes", } -export interface Status { - observedGeneration?: number; - phase?: Phase; - retryAttempt?: number; - titles?: string[]; -} - -export enum Phase { - Failed = "Failed", - Pending = "Pending", - Ready = "Ready", -} - RegisterKind(Exemption, { group: "uds.dev", version: "v1alpha1", diff --git a/src/pepr/operator/crd/index.ts b/src/pepr/operator/crd/index.ts index 959ae0029..163b8387a 100644 --- a/src/pepr/operator/crd/index.ts +++ b/src/pepr/operator/crd/index.ts @@ -12,8 +12,6 @@ export { } from "./generated/package-v1alpha1"; export { - Phase as ExemptPhase, - Status as ExemptStatus, ExemptionElement, Matcher, Kind as MatcherKind, diff --git a/src/pepr/operator/crd/register.ts b/src/pepr/operator/crd/register.ts index b7c49db0b..92782bfe6 100644 --- a/src/pepr/operator/crd/register.ts +++ b/src/pepr/operator/crd/register.ts @@ -3,69 +3,74 @@ import { K8s, Log, kind } from "pepr"; import { v1alpha1 as exemption } from "./sources/exemption/v1alpha1"; import { v1alpha1 as pkg } from "./sources/package/v1alpha1"; -// Register the CRD if we're in watch or dev mode -if (process.env.PEPR_WATCH_MODE === "true" || process.env.PEPR_MODE === "dev") { - K8s(kind.CustomResourceDefinition) - .Apply( - { - apiVersion: "apiextensions.k8s.io/v1", - kind: "CustomResourceDefinition", - metadata: { - name: "packages.uds.dev", - }, - spec: { - group: "uds.dev", - versions: [pkg], - scope: "Namespaced", - names: { - plural: "packages", - singular: "package", - kind: "Package", - shortNames: ["pkg"], +export async function registerCRDs() { + // Register the Package CRD if we're in watch or dev mode + if (process.env.PEPR_WATCH_MODE === "true" || process.env.PEPR_MODE === "dev") { + await K8s(kind.CustomResourceDefinition) + .Apply( + { + apiVersion: "apiextensions.k8s.io/v1", + kind: "CustomResourceDefinition", + metadata: { + name: "packages.uds.dev", + }, + spec: { + group: "uds.dev", + versions: [pkg], + scope: "Namespaced", + names: { + plural: "packages", + singular: "package", + kind: "Package", + shortNames: ["pkg"], + }, }, }, - }, - { force: true }, - ) - .then(() => { - Log.info("CRD registered"); - }) - .catch(err => { - Log.error({ err }, "Failed to register CRD"); + { force: true }, + ) + .then(() => { + Log.info("Package CRD registered"); + }) + .catch(err => { + Log.error({ err }, "Failed to register Package CRD"); - // Sad times, let's exit - process.exit(1); - }); + // Sad times, let's exit + process.exit(1); + }); + } - K8s(kind.CustomResourceDefinition) - .Apply( - { - apiVersion: "apiextensions.k8s.io/v1", - kind: "CustomResourceDefinition", - metadata: { - name: "exemptions.uds.dev", - }, - spec: { - group: "uds.dev", - versions: [exemption], - scope: "Namespaced", - names: { - plural: "exemptions", - singular: "exemption", - kind: "Exemption", - shortNames: ["exempt"], + // Register the Exemption CRD if we're in "admission" or dev mode (Exemptions are watched by the admission controllers) + if (process.env.PEPR_WATCH_MODE === "false" || process.env.PEPR_MODE === "dev") { + await K8s(kind.CustomResourceDefinition) + .Apply( + { + apiVersion: "apiextensions.k8s.io/v1", + kind: "CustomResourceDefinition", + metadata: { + name: "exemptions.uds.dev", + }, + spec: { + group: "uds.dev", + versions: [exemption], + scope: "Namespaced", + names: { + plural: "exemptions", + singular: "exemption", + kind: "Exemption", + shortNames: ["exempt"], + }, }, }, - }, - { force: true }, - ) - .then(() => { - Log.info("Exemption CRD registered"); - }) - .catch(err => { - Log.error(err); + { force: true }, + ) + .then(() => { + Log.info("Exemption CRD registered"); + }) + .catch(err => { + Log.error({ err }, "Failed to register Exemption CRD"); - // Sad times, let's exit - process.exit(1); - }); + // Sad times, let's exit + process.exit(1); + }); + } } diff --git a/src/pepr/operator/crd/sources/exemption/v1alpha1.ts b/src/pepr/operator/crd/sources/exemption/v1alpha1.ts index 16470e17c..71ff236f2 100644 --- a/src/pepr/operator/crd/sources/exemption/v1alpha1.ts +++ b/src/pepr/operator/crd/sources/exemption/v1alpha1.ts @@ -4,59 +4,17 @@ export const v1alpha1: V1CustomResourceDefinitionVersion = { name: "v1alpha1", served: true, storage: true, - additionalPrinterColumns: [ - { - name: "Status", - type: "string", - description: "The status of the exemption", - jsonPath: ".status.phase", - }, - { - name: "Exemptions", - type: "string", - description: "Titles of the exemptions", - jsonPath: ".status.titles", - }, - { - name: "Age", - type: "date", - description: "The age of the exemption", - jsonPath: ".metadata.creationTimestamp", - }, - ], - subresources: { - status: {}, - }, schema: { openAPIV3Schema: { type: "object", properties: { - status: { - type: "object", - properties: { - observedGeneration: { - type: "integer", - }, - phase: { - enum: ["Pending", "Ready", "Failed"], - type: "string", - }, - titles: { - type: "array", - items: { - type: "string", - }, - }, - retryAttempt: { - type: "integer", - }, - }, - } as V1JSONSchemaProps, spec: { type: "object", + required: ["exemptions"], properties: { exemptions: { type: "array", + minItems: 1, description: "Policy exemptions", items: { type: "object", diff --git a/src/pepr/operator/crd/validators/exempt-validator.spec.ts b/src/pepr/operator/crd/validators/exempt-validator.spec.ts index 9b042cd28..1ce8d1cbf 100644 --- a/src/pepr/operator/crd/validators/exempt-validator.spec.ts +++ b/src/pepr/operator/crd/validators/exempt-validator.spec.ts @@ -64,14 +64,6 @@ describe("Test validation of Exemption CRs", () => { expect(mockReq.Approve).toHaveBeenCalledTimes(1); }); - it("checks for at least 1 exemption block", async () => { - const mockReq = makeMockReq({ exempts: [] }); - await exemptValidator(mockReq); - expect(mockReq.Deny).toHaveBeenLastCalledWith( - "Invalid number of exemptions: must have at least 1", - ); - }); - it("denies matcher regex patterns with leading and trailing slashes", async () => { const wrongMatcherName = "/^neuvector-enforcer-pod*/"; const mockReq = makeMockReq({ diff --git a/src/pepr/operator/crd/validators/exempt-validator.ts b/src/pepr/operator/crd/validators/exempt-validator.ts index 89603477d..06c9fbc68 100644 --- a/src/pepr/operator/crd/validators/exempt-validator.ts +++ b/src/pepr/operator/crd/validators/exempt-validator.ts @@ -30,11 +30,6 @@ export async function exemptValidator(req: PeprValidateRequest) { } } - // Validate there's at least 1 exemption element - if (exemptions.length === 0) { - return req.Deny("Invalid number of exemptions: must have at least 1"); - } - // Validate exemption element policies and matcher kind are compatible for (const e of exemptions) { const policies = kindToPolicyMap.get(e.matcher.kind!)!; diff --git a/src/pepr/operator/index.ts b/src/pepr/operator/index.ts index 7c9909c78..0555c5ab2 100644 --- a/src/pepr/operator/index.ts +++ b/src/pepr/operator/index.ts @@ -3,7 +3,6 @@ import { a } from "pepr"; import { When } from "./common"; // Controller imports -import { removeExemptions } from "./controllers/exemptions/exemptions"; import { cleanupNamespace } from "./controllers/istio/injection"; import { purgeSSOClients } from "./controllers/keycloak/client-sync"; import { @@ -14,11 +13,10 @@ import { // CRD imports import { UDSExemption, UDSPackage } from "./crd"; -import { exemptValidator } from "./crd/validators/exempt-validator"; import { validator } from "./crd/validators/package-validator"; // Reconciler imports -import { exemptReconciler } from "./reconcilers/exempt-reconciler"; +import { exemptValidator } from "./crd/validators/exempt-validator"; import { packageReconciler } from "./reconcilers/package-reconciler"; // Export the operator capability for registration in the root pepr.ts @@ -61,8 +59,5 @@ When(UDSPackage) // Enqueue the package for processing .Reconcile(packageReconciler); -//Watch for changes to the UDSExemption CRD and cleanup exemptions in policies Store -When(UDSExemption).IsDeleted().Watch(removeExemptions); - -// Watch for changes to the UDSExemption CRD to enqueue an exemption for processing -When(UDSExemption).IsCreatedOrUpdated().Validate(exemptValidator).Reconcile(exemptReconciler); +// Watch for Exemptions and validate +When(UDSExemption).IsCreatedOrUpdated().Validate(exemptValidator); diff --git a/src/pepr/operator/reconcilers/exempt-reconciler.ts b/src/pepr/operator/reconcilers/exempt-reconciler.ts deleted file mode 100644 index c1f772ef8..000000000 --- a/src/pepr/operator/reconcilers/exempt-reconciler.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Log } from "pepr"; - -import { handleFailure, shouldSkip, uidSeen, updateStatus } from "."; -import { processExemptions } from "../controllers/exemptions/exemptions"; -import { Phase, UDSExemption } from "../crd"; - -export async function exemptReconciler(exempt: UDSExemption) { - if (shouldSkip(exempt)) { - return; - } - - const metadata = exempt.metadata!; - const { namespace, name } = metadata; - - Log.debug(exempt, `Processing Exemption ${namespace}/${name}`); - - try { - // Mark the exemption as pending - await updateStatus(exempt, { phase: Phase.Pending }); - - // Process the exemptions - processExemptions(exempt); - - // Mark the exemption as ready - await updateStatus(exempt, { - phase: Phase.Ready, - observedGeneration: metadata.generation, - titles: exempt.spec?.exemptions?.map(e => e.title || e.matcher.name), - }); - - // Update to indicate this version of pepr-core has reconciled the package successfully once - uidSeen.add(exempt.metadata!.uid!); - } catch (err) { - // Handle the failure - void handleFailure(err, exempt); - } -} diff --git a/src/pepr/operator/reconcilers/index.spec.ts b/src/pepr/operator/reconcilers/index.spec.ts index db4fa6c39..ce408c04e 100644 --- a/src/pepr/operator/reconcilers/index.spec.ts +++ b/src/pepr/operator/reconcilers/index.spec.ts @@ -4,7 +4,7 @@ import { K8s, Log, kind } from "pepr"; import { Mock } from "jest-mock"; import { handleFailure, shouldSkip, uidSeen, updateStatus, writeEvent } from "."; -import { ExemptStatus, Phase, PkgStatus, UDSExemption, UDSPackage } from "../crd"; +import { Phase, PkgStatus, UDSPackage } from "../crd"; jest.mock("pepr", () => ({ K8s: jest.fn(), @@ -78,17 +78,6 @@ describe("updateStatus", () => { status, }); }); - - it("should update the status of an exemption", async () => { - const cr = { kind: "Exemption", metadata: { name: "test", namespace: "default" } }; - const status = { phase: Phase.Ready }; - await updateStatus(cr as GenericKind, status as ExemptStatus); - expect(K8s).toHaveBeenCalledWith(UDSExemption); - expect(PatchStatus).toHaveBeenCalledWith({ - metadata: { name: "test", namespace: "default" }, - status, - }); - }); }); describe("writeEvent", () => { @@ -150,7 +139,7 @@ describe("handleFailure", () => { it("should handle a 404 error", async () => { const err = { status: 404, message: "Not found" }; const cr = { metadata: { namespace: "default", name: "test" } }; - await handleFailure(err, cr as UDSPackage | UDSExemption); + await handleFailure(err, cr as UDSPackage); expect(Log.warn).toHaveBeenCalledWith({ err }, "Package metadata seems to have been deleted"); expect(Create).not.toHaveBeenCalled(); }); @@ -162,7 +151,7 @@ describe("handleFailure", () => { apiVersion: "v1", metadata: { namespace: "default", name: "test", generation: 1, uid: "1" }, }; - await handleFailure(err, cr as UDSPackage | UDSExemption); + await handleFailure(err, cr as UDSPackage); expect(Log.error).toHaveBeenCalledWith( { err }, "Reconciliation attempt 1 failed for default/test, retrying...", @@ -204,7 +193,7 @@ describe("handleFailure", () => { metadata: { namespace: "default", name: "test", generation: 1, uid: "1" }, status: { phase: Phase.Pending, retryAttempt: 5 }, }; - await handleFailure(err, cr as UDSPackage | UDSExemption); + await handleFailure(err, cr as UDSPackage); expect(Log.error).toHaveBeenCalledWith( { err }, "Error configuring default/test, maxed out retries", diff --git a/src/pepr/operator/reconcilers/index.ts b/src/pepr/operator/reconcilers/index.ts index 5259215a2..4d07a3d27 100644 --- a/src/pepr/operator/reconcilers/index.ts +++ b/src/pepr/operator/reconcilers/index.ts @@ -1,7 +1,6 @@ -import { GenericKind } from "kubernetes-fluent-client"; import { K8s, Log, kind } from "pepr"; -import { ExemptStatus, Phase, PkgStatus, UDSExemption, UDSPackage } from "../crd"; +import { Phase, PkgStatus, UDSPackage } from "../crd"; import { Status } from "../crd/generated/package-v1alpha1"; export const uidSeen = new Set(); @@ -12,7 +11,7 @@ export const uidSeen = new Set(); * @param cr The custom resource to check * @returns true if the CRD is pending or the current generation has been processed */ -export function shouldSkip(cr: UDSExemption | UDSPackage) { +export function shouldSkip(cr: UDSPackage) { const isPending = cr.status?.phase === Phase.Pending; const isCurrentGeneration = cr.metadata?.generation === cr.status?.observedGeneration; @@ -40,12 +39,11 @@ export function shouldSkip(cr: UDSExemption | UDSPackage) { * @param cr The custom resource to update * @param status The new status */ -export async function updateStatus(cr: GenericKind, status: PkgStatus | ExemptStatus) { - const model = cr.kind === "Package" ? UDSPackage : UDSExemption; +export async function updateStatus(cr: UDSPackage, status: PkgStatus) { Log.debug(cr.metadata, `Updating status to ${status.phase}`); // Update the status of the CRD - await K8s(model).PatchStatus({ + await K8s(UDSPackage).PatchStatus({ metadata: { name: cr.metadata!.name, namespace: cr.metadata!.namespace, @@ -61,7 +59,7 @@ export async function updateStatus(cr: GenericKind, status: PkgStatus | ExemptSt * @param message A human-readable message for the event * @param type The type of event to write */ -export async function writeEvent(cr: GenericKind, event: Partial) { +export async function writeEvent(cr: UDSPackage, event: Partial) { Log.debug(cr.metadata, `Writing event: ${event.message}`); await K8s(kind.CoreEvent).Create({ @@ -92,10 +90,7 @@ export async function writeEvent(cr: GenericKind, event: Partial * @param err The error-like object * @param cr The custom resource that failed */ -export async function handleFailure( - err: { status: number; message: string }, - cr: UDSPackage | UDSExemption, -) { +export async function handleFailure(err: { status: number; message: string }, cr: UDSPackage) { const metadata = cr.metadata!; const identifier = `${metadata.namespace}/${metadata.name}`; let status: Status; diff --git a/src/pepr/policies/common.ts b/src/pepr/policies/common.ts index 28d7a11b4..0a953faa4 100644 --- a/src/pepr/policies/common.ts +++ b/src/pepr/policies/common.ts @@ -13,7 +13,7 @@ export const policies = new Capability({ "Collection of core validation policies for Pods, ConfigMaps, and other Kubernetes resources.", }); -export const { Store, When } = policies; +export const { When } = policies; // Returns all volumes in the pod export function volumes(request: PeprValidateRequest) { diff --git a/src/pepr/policies/exemptions/index.spec.ts b/src/pepr/policies/exemptions/index.spec.ts index a9c753dc3..2ab36dd25 100644 --- a/src/pepr/policies/exemptions/index.spec.ts +++ b/src/pepr/policies/exemptions/index.spec.ts @@ -1,15 +1,20 @@ import { beforeAll, describe, expect, it, jest } from "@jest/globals"; +import { PeprValidateRequest, kind } from "pepr"; import { isExempt } from "."; -import { PeprValidateRequest } from "pepr"; -import { kind } from "pepr"; -import { Policy } from "../../operator/crd"; -import { Store } from "../common"; +import { ExemptionStore } from "../../operator/controllers/exemptions/exemption-store"; +import { MatcherKind, Policy } from "../../operator/crd"; describe("test registering exemptions", () => { beforeAll(() => { - jest - .spyOn(Store, "getItem") - .mockReturnValue('[{"namespace": "neuvector", "name": "^neuvector-enforcer-pod-.*"}]'); + ExemptionStore.init(); + jest.spyOn(ExemptionStore, "getByPolicy").mockReturnValue([ + { + namespace: "neuvector", + name: "^neuvector-enforcer-pod-.*", + kind: MatcherKind.Pod, + owner: "uid", + }, + ]); }); it("should be exempt", () => { diff --git a/src/pepr/policies/exemptions/index.ts b/src/pepr/policies/exemptions/index.ts index 9fd2b99a3..8eb2ca73c 100644 --- a/src/pepr/policies/exemptions/index.ts +++ b/src/pepr/policies/exemptions/index.ts @@ -1,7 +1,7 @@ import { KubernetesObject } from "kubernetes-fluent-client"; import { Log, PeprMutateRequest, PeprValidateRequest } from "pepr"; +import { ExemptionStore } from "../../operator/controllers/exemptions/exemption-store"; import { Policy } from "../../operator/crd"; -import { Store } from "../common"; /** * Check a resource against an exemption list for use by the validation action. @@ -14,30 +14,38 @@ export function isExempt( request: PeprValidateRequest | PeprMutateRequest, policy: Policy, ) { - const exemptList = JSON.parse(Store.getItem(policy) || "[]"); + const exemptList = ExemptionStore.getByPolicy(policy); + const resourceName = request.Raw.metadata?.name || request.Raw.metadata?.generateName; + const resourceNamespace = request.Raw.metadata?.namespace; - // Loop through the exempt list - for (const exempt of exemptList) { - // If the exempt namespace is specified, check it - if (exempt.namespace && exempt.namespace !== request.Raw.metadata?.namespace) { - continue; - } + if (exemptList.length != 0) { + // Debug log to provide current exemptions for policy + Log.debug( + `Checking ${resourceName} against ${policy} exemptions: ${JSON.stringify(exemptList)}`, + ); + for (const exempt of exemptList) { + // If the exempt namespace is specified, check it + if (exempt.namespace !== resourceNamespace) { + continue; + } - // If the exempt name is specified, check it - const name = request.Raw.metadata?.name || request.Raw.metadata?.generateName; - if (exempt.name && !name?.match(exempt.name)) { - continue; - } + // If the exempt name is specified, check it + if (!resourceName?.match(exempt.name)) { + continue; + } - // If we get here, the request is exempt - Log.info("request is exempt", { exempt }); - return true; + // If we get here, the request is exempt + Log.info(`${resourceName} is exempt from ${policy}`); + return true; + } } // No exemptions matched return false; } +export const exemptionAnnotationPrefix = "uds-core.pepr.dev/uds-core-policies"; + /** * * @param policy @@ -46,7 +54,7 @@ export function isExempt( export function markExemption(policy: Policy) { return (request: PeprMutateRequest) => { if (isExempt(request, policy)) { - request.SetAnnotation(`uds-core.pepr.dev/uds-core-policies.${policy}`, "exempted"); + request.SetAnnotation(`${exemptionAnnotationPrefix}.${policy}`, "exempted"); return; } }; diff --git a/src/pepr/policies/index.ts b/src/pepr/policies/index.ts index 6074b9da5..0b8d5a1b8 100644 --- a/src/pepr/policies/index.ts +++ b/src/pepr/policies/index.ts @@ -1,6 +1,30 @@ // Various validation actions for Kubernetes resources from Big Bang +import { K8s, Log } from "pepr"; +import { ExemptionStore } from "../operator/controllers/exemptions/exemption-store"; +import { processExemptions } from "../operator/controllers/exemptions/exemptions"; +import { Matcher, Policy, UDSExemption } from "../operator/crd"; +import "./networking"; import "./security"; import "./storage"; -import "./networking"; export { policies } from "./common"; + +export type StoredMatcher = Matcher & { owner: string }; +export type PolicyMap = Map; + +export async function startExemptionWatch() { + ExemptionStore.init(); + + // only run in admission controller or dev mode + if (process.env.PEPR_WATCH_MODE === "false" || process.env.PEPR_MODE === "dev") { + const watcher = K8s(UDSExemption).Watch(async (exemption, phase) => { + Log.debug(`Processing exemption ${exemption.metadata?.name}, watch phase: ${phase}`); + + processExemptions(exemption, phase); + }); + + // This will run until the process is terminated or the watch is aborted + Log.debug("Starting exemption watch..."); + await watcher.start(); + } +} diff --git a/src/pepr/policies/security.ts b/src/pepr/policies/security.ts index a87a4af34..e4b782433 100644 --- a/src/pepr/policies/security.ts +++ b/src/pepr/policies/security.ts @@ -9,7 +9,7 @@ import { securityContextContainers, securityContextMessage, } from "./common"; -import { isExempt, markExemption } from "./exemptions"; +import { exemptionAnnotationPrefix, isExempt, markExemption } from "./exemptions"; /** * This policy ensures that Pods do not allow privilege escalation. @@ -61,7 +61,7 @@ When(a.Pod) .IsCreatedOrUpdated() .Mutate(request => { markExemption(Policy.RequireNonRootUser)(request); - if (request.HasAnnotation(`uds-core.pepr.dev/uds-core-policies.${Policy.RequireNonRootUser}`)) { + if (request.HasAnnotation(`${exemptionAnnotationPrefix}.${Policy.RequireNonRootUser}`)) { return; } @@ -314,9 +314,7 @@ When(a.Pod) .IsCreatedOrUpdated() .Mutate(request => { markExemption(Policy.DropAllCapabilities)(request); - if ( - request.HasAnnotation(`uds-core.pepr.dev/uds-core-policies.${Policy.DropAllCapabilities}`) - ) { + if (request.HasAnnotation(`${exemptionAnnotationPrefix}.${Policy.DropAllCapabilities}`)) { return; } diff --git a/src/test/chart/.helmignore b/src/test/chart/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/src/test/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/src/test/chart/Chart.yaml b/src/test/chart/Chart.yaml new file mode 100644 index 000000000..5cabd1306 --- /dev/null +++ b/src/test/chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: exempted-app +description: A Helm chart for testing an exempted-app +type: application +version: 0.1.0 diff --git a/src/test/chart/templates/exemption1.yaml b/src/test/chart/templates/exemption1.yaml new file mode 100644 index 000000000..b170dc955 --- /dev/null +++ b/src/test/chart/templates/exemption1.yaml @@ -0,0 +1,13 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: podinfo1 + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DisallowPrivileged + title: "podinfo1" + matcher: + namespace: podinfo + name: "^podinfo.*" diff --git a/src/test/chart/templates/exemption2.yaml b/src/test/chart/templates/exemption2.yaml new file mode 100644 index 000000000..6e03d00c0 --- /dev/null +++ b/src/test/chart/templates/exemption2.yaml @@ -0,0 +1,13 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: podinfo2 + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - RequireNonRootUser + title: "podinfo2" + matcher: + namespace: podinfo + name: "^podinfo.*" diff --git a/src/test/chart/templates/exemption3.yaml b/src/test/chart/templates/exemption3.yaml new file mode 100644 index 000000000..6e470070e --- /dev/null +++ b/src/test/chart/templates/exemption3.yaml @@ -0,0 +1,13 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: podinfo3 + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DropAllCapabilities + title: "podinfo3" + matcher: + namespace: podinfo + name: "^podinfo.*" diff --git a/src/test/chart/templates/exemption4.yaml b/src/test/chart/templates/exemption4.yaml new file mode 100644 index 000000000..7c71840c5 --- /dev/null +++ b/src/test/chart/templates/exemption4.yaml @@ -0,0 +1,14 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: podinfo4 + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DisallowNodePortServices + title: "podinfo4" + matcher: + namespace: podinfo + name: "^podinfo.*" + kind: service diff --git a/src/test/chart/templates/exemption5.yaml b/src/test/chart/templates/exemption5.yaml new file mode 100644 index 000000000..856688656 --- /dev/null +++ b/src/test/chart/templates/exemption5.yaml @@ -0,0 +1,13 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: podinfo5 + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - RestrictHostPorts + title: "podinfo1" + matcher: + namespace: podinfo + name: "^podinfo.*" diff --git a/src/test/chart/values.yaml b/src/test/chart/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/podinfo-values.yaml b/src/test/podinfo-values.yaml new file mode 100644 index 000000000..7be05fd31 --- /dev/null +++ b/src/test/podinfo-values.yaml @@ -0,0 +1,11 @@ +# Values set to intentionally violate pepr policies +securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + +service: + enabled: true + type: NodePort + nodePort: 31123 + hostPort: 1000 diff --git a/src/test/tasks.yaml b/src/test/tasks.yaml index 2166e70cb..0a03c5324 100644 --- a/src/test/tasks.yaml +++ b/src/test/tasks.yaml @@ -4,7 +4,8 @@ tasks: actions: - description: Create zarf package for the test resources cmd: "uds zarf package create src/test --confirm --no-progress" - - description: Apply the test resources + + - description: Deploy the test resources cmd: "uds zarf package deploy build/zarf-package-uds-core-test-apps-*.zst --confirm --no-progress" - description: Wait for the admin app to be ready @@ -49,5 +50,13 @@ tasks: address: demo.uds.dev/status/202 code: 202 + - description: Verify podinfo is healthy + wait: + cluster: + kind: Pod + name: app.kubernetes.io/name=podinfo + namespace: podinfo + condition: Ready + - description: Remove the test resources - cmd: "uds zarf package remove build/zarf-package-uds-core-test-apps-*.zst --confirm" + cmd: "uds zarf package remove build/zarf-package-uds-core-test-apps-*.zst --confirm --no-progress" diff --git a/src/test/zarf.yaml b/src/test/zarf.yaml index 4d9ada5ec..6d58271c6 100644 --- a/src/test/zarf.yaml +++ b/src/test/zarf.yaml @@ -14,5 +14,23 @@ components: - name: app-tenant files: - "app-tenant.yaml" + images: - docker.io/kong/httpbin:latest + + - name: podinfo + required: true + charts: + - name: exempted-app + namespace: exempted-app + localPath: ./chart + version: 0.1.0 + - name: podinfo + version: 6.4.0 + namespace: podinfo + url: https://github.com/stefanprodan/podinfo.git + gitPath: charts/podinfo + valuesFiles: + - ./podinfo-values.yaml + images: + - ghcr.io/stefanprodan/podinfo:6.4.0 diff --git a/tasks.yaml b/tasks.yaml index e01ce4443..d6976988a 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -27,11 +27,11 @@ tasks: task: setup:create-k3d-cluster - description: "Deploy the Istio source package with Zarf Dev" - cmd: "uds zarf dev deploy src/istio --flavor ${FLAVOR}" + cmd: "uds zarf dev deploy src/istio --flavor ${FLAVOR} --no-progress" # Note, this abuses the --flavor flag to only install the CRDs from this package - the "crds-only" flavor is not an explicit flavor of the package - description: "Deploy the Prometheus-Stack source package with Zarf Dev to only install the CRDs" - cmd: "uds zarf dev deploy src/prometheus-stack --flavor crds-only" + cmd: "uds zarf dev deploy src/prometheus-stack --flavor crds-only --no-progress" - description: "Dev instructions" cmd: |