initial external data integration for validation (open-policy-agent#1573

)

Signed-off-by: Rob Mason <rob.mason@borumborad.com>

Makefile

@@ -6,6 +6,7 @@ CRD_IMG := $(CRD_REPOSITORY):latest
 # DEV_TAG will be replaced with short Git SHA on pre-release stage in CI
 DEV_TAG ?= dev
 USE_LOCAL_IMG ?= false
+ENABLE_EXTERNAL_DATA ?= false
 
 VERSION := v3.7.0-beta.1
 
@@ -71,9 +72,6 @@ MANAGER_IMAGE_PATCH := "apiVersion: apps/v1\
 \n        - --operation=status\
 \n        - --logtostderr"
 
-
-FRAMEWORK_PACKAGE := github.com/open-policy-agent/frameworks/constraint
-
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -191,6 +189,9 @@ deploy-mutation: patch-image
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: patch-image manifests
+ifeq ($(ENABLE_EXTERNAL_DATA),true)
+	@grep -q -v 'enable-external-data' ./config/overlays/dev/manager_image_patch.yaml && sed -i '/- --operation=webhook/a \ \ \ \ \ \ \ \ - --enable-external-data=true' ./config/overlays/dev/manager_image_patch.yaml
+endif
 	docker run -v $(shell pwd)/config:/config -v $(shell pwd)/vendor:/vendor \
 	  k8s.gcr.io/kustomize/kustomize:v${KUSTOMIZE_VERSION} build \
 	  /config/overlays/dev | kubectl apply -f -

vendor/github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/zz_generated.defaults.go → apis/addtoscheme_provider_v1alpha1.go

@@ -1,5 +1,3 @@
-// +build !ignore_autogenerated
-
 /*
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-// Code generated by defaulter-gen. DO NOT EDIT.
 
-package templates
+package apis
 
 import (
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata/v1alpha1"
 )
 
-// RegisterDefaults adds defaulters functions to the given scheme.
-// Public to allow building arbitrary schemes.
-// All generated defaulters are covering - they call all nested defaulters.
-func RegisterDefaults(scheme *runtime.Scheme) error {
-	return nil
+func init() {
+	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
+	AddToSchemes = append(AddToSchemes, v1alpha1.AddToScheme)
 }

cmd/build/helmify/kustomization.yaml

@@ -58,6 +58,12 @@ patchesJson6902:
       kind: CustomResourceDefinition
       name: modifyset.mutations.gatekeeper.sh
     path: labels_patch.yaml
+  - target:
+      group: apiextensions.k8s.io
+      version: v1
+      kind: CustomResourceDefinition
+      name: providers.externaldata.gatekeeper.sh
+    path: labels_patch.yaml
   # these are defined in the chart values rather than hard-coded
   - target:
       kind: Deployment

cmd/build/helmify/kustomize-for-helm.yaml

@@ -48,6 +48,12 @@ metadata:
   name: assign.mutations.gatekeeper.sh
 status: null
 ---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: providers.externaldata.gatekeeper.sh
+status: null
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:

config/rbac/role.yaml

@@ -58,6 +58,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - externaldata.gatekeeper.sh
+  resources:
+  - providers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - mutations.gatekeeper.sh
   resources:

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.16.0
 	github.com/open-policy-agent/cert-controller v0.2.0
-	github.com/open-policy-agent/frameworks/constraint v0.0.0-20210816184142-2924b2c86f76
+	github.com/open-policy-agent/frameworks/constraint v0.0.0-20211012225819-a8579b618cb7
 	github.com/open-policy-agent/opa v0.29.4
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0

go.sum

@@ -607,8 +607,8 @@ github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/open-policy-agent/cert-controller v0.2.0 h1:Z+IPOYDor28l6cjEo2WvTZY6Bv5oYR6wECEIP8pyG/M=
 github.com/open-policy-agent/cert-controller v0.2.0/go.mod h1:SWS7Ame8oKHF11cDsQCFlULrrOMV5Z59FIGEAF/M6YI=
-github.com/open-policy-agent/frameworks/constraint v0.0.0-20210816184142-2924b2c86f76 h1:T6XdkOS3onLqKniqlqImmlSct46oMl50IDZ/YSrHVQE=
-github.com/open-policy-agent/frameworks/constraint v0.0.0-20210816184142-2924b2c86f76/go.mod h1:sxECOn2E9o4DIK6ttinq1frfiErxi0Z8oIgtz7VDVBc=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20211012225819-a8579b618cb7 h1:X2dfeC/XNGcoBlytXDKnlom0roCCM8bS75Ms+vqDe/s=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20211012225819-a8579b618cb7/go.mod h1:sxECOn2E9o4DIK6ttinq1frfiErxi0Z8oIgtz7VDVBc=
 github.com/open-policy-agent/opa v0.29.4 h1:rNa/Gd3Fs0xWgL0aZoyblRwCZLJsSLDQOhnck6DWpaA=
 github.com/open-policy-agent/opa v0.29.4/go.mod h1:ZCOTD3yyFR8JvF8ETdWdiSPn9WcF1dXeQWOv7VoPorU=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=

main.go

@@ -28,13 +28,15 @@ import (
 	"github.com/open-policy-agent/cert-controller/pkg/rotator"
 	opa "github.com/open-policy-agent/frameworks/constraint/pkg/client"
 	"github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/local"
+	frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata"
 	api "github.com/open-policy-agent/gatekeeper/apis"
 	configv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
 	mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1alpha1"
 	statusv1beta1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1"
 	"github.com/open-policy-agent/gatekeeper/pkg/audit"
 	"github.com/open-policy-agent/gatekeeper/pkg/controller"
 	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
+	"github.com/open-policy-agent/gatekeeper/pkg/externaldata"
 	"github.com/open-policy-agent/gatekeeper/pkg/metrics"
 	"github.com/open-policy-agent/gatekeeper/pkg/mutation"
 	"github.com/open-policy-agent/gatekeeper/pkg/operations"
@@ -205,7 +207,7 @@ func main() {
 	sw := watch.NewSwitch()
 
 	// Setup tracker and register readiness probe.
-	tracker, err := readiness.SetupTracker(mgr, *mutation.MutationEnabled)
+	tracker, err := readiness.SetupTracker(mgr, *mutation.MutationEnabled, *externaldata.ExternalDataEnabled)
 	if err != nil {
 		setupLog.Error(err, "unable to register readiness tracker")
 		os.Exit(1)
@@ -242,8 +244,15 @@ func setupControllers(mgr ctrl.Manager, sw *watch.ControllerSwitch, tracker *rea
 	// Block until the setup (certificate generation) finishes.
 	<-setupFinished
 
+	var providerCache *frameworksexternaldata.ProviderCache
+	args := []local.Arg{local.Tracing(false), local.DisableBuiltins(disabledBuiltins.ToSlice()...)}
+	if *externaldata.ExternalDataEnabled {
+		providerCache = frameworksexternaldata.NewCache()
+		args = append(args, local.AddExternalDataProviderCache(providerCache))
+	}
 	// initialize OPA
-	driver := local.New(local.Tracing(false), local.DisableBuiltins(disabledBuiltins.ToSlice()...))
+	driver := local.New(args...)
+
 	backend, err := opa.NewBackend(opa.Driver(driver))
 	if err != nil {
 		setupLog.Error(err, "unable to set up OPA backend")
@@ -287,6 +296,7 @@ func setupControllers(mgr ctrl.Manager, sw *watch.ControllerSwitch, tracker *rea
 		Tracker:          tracker,
 		ProcessExcluder:  processExcluder,
 		MutationSystem:   mutationSystem,
+		ProviderCache:    providerCache,
 	}
 
 	ctx := context.Background()

manifest_staging/charts/gatekeeper/crds/constrainttemplate-customresourcedefinition.yaml

@@ -30,7 +30,7 @@ spec:
           metadata:
             type: object
           spec:
-            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate
+            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate.
             properties:
               crd:
                 properties:
@@ -73,7 +73,7 @@ spec:
                 type: array
             type: object
           status:
-            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate
+            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate.
             properties:
               byPod:
                 items:
@@ -125,7 +125,7 @@ spec:
           metadata:
             type: object
           spec:
-            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate
+            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate.
             properties:
               crd:
                 properties:
@@ -168,7 +168,7 @@ spec:
                 type: array
             type: object
           status:
-            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate
+            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate.
             properties:
               byPod:
                 items:
@@ -220,7 +220,7 @@ spec:
           metadata:
             type: object
           spec:
-            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate
+            description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate.
             properties:
               crd:
                 properties:
@@ -263,7 +263,7 @@ spec:
                 type: array
             type: object
           status:
-            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate
+            description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate.
             properties:
               byPod:
                 items:

manifest_staging/charts/gatekeeper/crds/provider-customresourcedefinition.yaml

@@ -0,0 +1,44 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.5.0
+  labels:
+    gatekeeper.sh/system: "yes"
+  name: providers.externaldata.gatekeeper.sh
+spec:
+  group: externaldata.gatekeeper.sh
+  names:
+    kind: Provider
+    listKind: ProviderList
+    plural: providers
+    singular: provider
+  preserveUnknownFields: false
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Provider is the Schema for the Provider API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the Provider specifications.
+            properties:
+              timeout:
+                description: Timeout is the timeout when querying the provider.
+                type: integer
+              url:
+                description: URL is the url for the provider. URL is prefixed with http:// or https://.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true

manifest_staging/charts/gatekeeper/templates/gatekeeper-manager-role-clusterrole.yaml

@@ -62,6 +62,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - externaldata.gatekeeper.sh
+  resources:
+  - providers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - mutations.gatekeeper.sh
   resources: