feat: add --image flag in gator test|expand (#2398)

* Add --image flag in gator test|expand Signed-off-by: davis-haba <davishaba@google.com> * remove debug make target Signed-off-by: davis-haba <davishaba@google.com> * lint Signed-off-by: davis-haba <davishaba@google.com> * throw away container logs /w test-gator-containerized Signed-off-by: davis-haba <davishaba@google.com> * lint Signed-off-by: davis-haba <davishaba@google.com> * avoid aufs and mount out container filesystem Signed-off-by: davis-haba <davishaba@google.com> * try mounting the volume from the host also Signed-off-by: davis-haba <davishaba@google.com> * docs for --image Signed-off-by: davis-haba <davishaba@google.com> * styling fixes Signed-off-by: davis-haba <davishaba@google.com> * newline when logging resource conflict in OCI images Signed-off-by: davis-haba <davishaba@google.com> * Refactor gator pkg Signed-off-by: davis-haba <davishaba@google.com> * fix formatting Signed-off-by: davis-haba <davishaba@google.com> * appease linter Signed-off-by: davis-haba <davishaba@google.com> * gator detect conflict on gknn not gvkn Signed-off-by: davis-haba <davishaba@google.com> * add more conflict tests, add docstrings Signed-off-by: davis-haba <davishaba@google.com> * appease linter Signed-off-by: davis-haba <davishaba@google.com> * consolidate docker images Signed-off-by: davis-haba <davishaba@google.com> * Update cmd/gator/expand/expand.go Co-authored-by: alex <8968914+acpana@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * Update cmd/gator/test/test.go Co-authored-by: alex <8968914+acpana@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * Update website/docs/gator.md Co-authored-by: alex <8968914+acpana@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * Update website/docs/gator.md Co-authored-by: alex <8968914+acpana@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * properly clean up files when image pull fails Signed-off-by: davis-haba <davishaba@google.com> * remove TODO comment Signed-off-by: davis-haba <davishaba@google.com> * update bundling section in gator Signed-off-by: davis-haba <davishaba@google.com> * resolve rebase issues Signed-off-by: davis-haba <davishaba@google.com> * Update test/gator/test/test.bats Co-authored-by: Sertaç Özercan <852750+sozercan@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * Update test/gator/expand/test.bats Co-authored-by: Sertaç Özercan <852750+sozercan@users.noreply.github.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> * address comments Signed-off-by: davis-haba <davishaba@google.com> * add note in docs that --image is alpha Signed-off-by: davis-haba <davishaba@google.com> Signed-off-by: davis-haba <52938648+davis-haba@users.noreply.github.com> Co-authored-by: alex <8968914+acpana@users.noreply.github.com> Co-authored-by: Sertaç Özercan <852750+sozercan@users.noreply.github.com>
open-policy-agent · Dec 14, 2022 · 3a04703 · 3a04703
1 parent 820a892
commit 3a04703
Show file tree

Hide file tree

Showing 515 changed files with 77,028 additions and 395 deletions.
diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml
@@ -105,7 +105,7 @@ jobs:
           make e2e-dependencies KUBERNETES_VERSION=${{ matrix.KUBERNETES_VERSION }}
 
       - name: gator test
-        run: make test-gator
+        run: make test-gator-containerized
 
   build_test:
     name: "Build and Test"

diff --git a/Makefile b/Makefile
@@ -18,6 +18,7 @@ KIND_VERSION ?= 0.17.0
 KUBERNETES_VERSION ?= 1.26.0
 KUSTOMIZE_VERSION ?= 3.8.9
 BATS_VERSION ?= 1.8.2
+ORAS_VERSION ?= 0.16.0
 BATS_TESTS_FILE ?= test/bats/test.bats
 HELM_VERSION ?= 3.7.2
 NODE_VERSION ?= 16-bullseye-slim
@@ -116,6 +117,11 @@ test-e2e:
 .PHONY: test-gator
 test-gator: gator test-gator-verify test-gator-test test-gator-expand
 
+.PHONY: test-gator-containerized
+test-gator-containerized: __test-image
+	docker run --privileged -v $(shell pwd):/app -v /var/lib/docker \
+	gatekeeper-test ./test/image/gator-test.sh
+
 .PHONY: test-gator-verify
 test-gator-verify: gator
 	./bin/gator verify test/gator/verify/suite.yaml
@@ -440,7 +446,10 @@ __tooling-image:
 
 __test-image:
 	docker build test/image \
-		-t gatekeeper-test
+		-t gatekeeper-test \
+		--build-arg YQ_VERSION=$(YQ_VERSION) \
+		--build-arg BATS_VERSION=$(BATS_VERSION) \
+		--build-arg ORAS_VERSION=$(ORAS_VERSION)
 
 .PHONY: vendor
 vendor:

diff --git a/cmd/gator/expand/expand.go b/cmd/gator/expand/expand.go
@@ -7,7 +7,8 @@ import (
 	"os"
 	"sort"
 
-	"github.com/open-policy-agent/gatekeeper/pkg/gator"
+	"github.com/open-policy-agent/gatekeeper/pkg/gator/expand"
+	"github.com/open-policy-agent/gatekeeper/pkg/gator/reader"
 	"github.com/spf13/cobra"
 
 	// yaml.v3 inserts a space before '-', which is inconsistent with standard
@@ -42,12 +43,16 @@ var (
 	flagFilenames []string
 	flagFormat    string
 	flagOutput    string
+	flagImages    []string
+	flagTempDir   string
 )
 
 const (
 	flagNameFilename = "filename"
 	flagNameFormat   = "format"
 	flagNameOutput   = "outputfile"
+	flagNameImage    = "image"
+	flagNameTempDir  = "tempdir"
 
 	stringJSON = "json"
 	stringYAML = "yaml"
@@ -59,18 +64,20 @@ func init() {
 	Cmd.Flags().StringArrayVarP(&flagFilenames, flagNameFilename, "n", []string{}, "a file or directory containing Kubernetes resources.  Can be specified multiple times.")
 	Cmd.Flags().StringVarP(&flagFormat, flagNameFormat, "f", "", fmt.Sprintf("Output format.  One of: %s|%s.", stringJSON, stringYAML))
 	Cmd.Flags().StringVarP(&flagOutput, flagNameOutput, "o", "", "Output file path. If the file already exists, it will be overwritten.")
+	Cmd.Flags().StringArrayVarP(&flagImages, flagNameImage, "i", []string{}, "a URL to an OCI image containing policies. Can be specified multiple times.")
+	Cmd.Flags().StringVarP(&flagTempDir, flagNameTempDir, "d", "", fmt.Sprintf("Specifies the temporary directory to download and unpack images to, if using the --%s flag. Optional.", flagNameImage))
 }
 
 func run(cmd *cobra.Command, args []string) {
-	unstrucs, err := gator.ReadSources(flagFilenames)
+	unstrucs, err := reader.ReadSources(flagFilenames, flagImages, flagTempDir)
 	if err != nil {
 		errFatalf("reading: %v\n", err)
 	}
 	if len(unstrucs) == 0 {
 		errFatalf("no input data identified\n")
 	}
 
-	resultants, err := gator.Expand(unstrucs)
+	resultants, err := expand.Expand(unstrucs)
 	if err != nil {
 		errFatalf("error expanding resources: %v", err)
 	}

diff --git a/cmd/gator/test/test.go b/cmd/gator/test/test.go
@@ -7,7 +7,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/open-policy-agent/gatekeeper/pkg/gator"
+	"github.com/open-policy-agent/gatekeeper/pkg/gator/reader"
 	"github.com/open-policy-agent/gatekeeper/pkg/gator/test"
 	"github.com/open-policy-agent/gatekeeper/pkg/util"
 	"github.com/spf13/cobra"
@@ -47,11 +47,15 @@ var (
 	flagFilenames    []string
 	flagOutput       string
 	flagIncludeTrace bool
+	flagImages       []string
+	flagTempDir      string
 )
 
 const (
 	flagNameFilename = "filename"
 	flagNameOutput   = "output"
+	flagNameImage    = "image"
+	flagNameTempDir  = "tempdir"
 
 	stringJSON          = "json"
 	stringYAML          = "yaml"
@@ -62,10 +66,12 @@ func init() {
 	Cmd.Flags().StringArrayVarP(&flagFilenames, flagNameFilename, "f", []string{}, "a file or directory containing Kubernetes resources.  Can be specified multiple times.")
 	Cmd.Flags().StringVarP(&flagOutput, flagNameOutput, "o", "", fmt.Sprintf("Output format.  One of: %s|%s.", stringJSON, stringYAML))
 	Cmd.Flags().BoolVarP(&flagIncludeTrace, "trace", "t", false, `include a trace for the underlying constraint framework evaluation`)
+	Cmd.Flags().StringArrayVarP(&flagImages, flagNameImage, "i", []string{}, "a URL to an OCI image containing policies. Can be specified multiple times.")
+	Cmd.Flags().StringVarP(&flagTempDir, flagNameTempDir, "d", "", fmt.Sprintf("Specifies the temporary directory to download and unpack images to, if using the --%s flag. Optional.", flagNameImage))
 }
 
 func run(cmd *cobra.Command, args []string) {
-	unstrucs, err := gator.ReadSources(flagFilenames)
+	unstrucs, err := reader.ReadSources(flagFilenames, flagImages, flagTempDir)
 	if err != nil {
 		errFatalf("reading: %v", err)
 	}

diff --git a/cmd/gator/verify/verify.go b/cmd/gator/verify/verify.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/open-policy-agent/gatekeeper/pkg/gator"
+	"github.com/open-policy-agent/gatekeeper/pkg/gator/verify"
 	"github.com/spf13/cobra"
 )
 
@@ -93,27 +94,27 @@ func runE(cmd *cobra.Command, args []string) error {
 	}
 	targetPath = strings.Trim(targetPath, "/")
 
-	suites, err := gator.ReadSuites(fileSystem, targetPath, originalPath, recursive)
+	suites, err := verify.ReadSuites(fileSystem, targetPath, originalPath, recursive)
 	if err != nil {
 		return fmt.Errorf("listing test files: %w", err)
 	}
-	filter, err := gator.NewFilter(run)
+	filter, err := verify.NewFilter(run)
 	if err != nil {
 		return fmt.Errorf("compiling filter: %w", err)
 	}
 
 	return runSuites(cmd.Context(), fileSystem, suites, filter)
 }
 
-func runSuites(ctx context.Context, fileSystem fs.FS, suites []*gator.Suite, filter gator.Filter) error {
+func runSuites(ctx context.Context, fileSystem fs.FS, suites []*verify.Suite, filter verify.Filter) error {
 	isFailure := false
 
-	runner, err := gator.NewRunner(fileSystem, gator.NewOPAClient, gator.IncludeTrace(includeTrace))
+	runner, err := verify.NewRunner(fileSystem, gator.NewOPAClient, verify.IncludeTrace(includeTrace))
 	if err != nil {
 		return err
 	}
 
-	results := make([]gator.SuiteResult, len(suites))
+	results := make([]verify.SuiteResult, len(suites))
 	i := 0
 
 	for _, suite := range suites {
@@ -133,7 +134,7 @@ func runSuites(ctx context.Context, fileSystem fs.FS, suites []*gator.Suite, fil
 		i++
 	}
 	w := &strings.Builder{}
-	printer := gator.PrinterGo{}
+	printer := verify.PrinterGo{}
 	err = printer.Print(w, results, verbose)
 	if err != nil {
 		return err

diff --git a/go.mod b/go.mod
@@ -46,13 +46,15 @@ require (
 	k8s.io/client-go v0.24.9
 	k8s.io/klog/v2 v2.70.1
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
+	oras.land/oras-go v1.2.1
 	sigs.k8s.io/controller-runtime v0.12.3
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	cloud.google.com/go/compute v1.6.1 // indirect
 	cloud.google.com/go/monitoring v1.5.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
@@ -62,6 +64,14 @@ require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containerd/containerd v1.6.10 // indirect
+	github.com/docker/cli v20.10.17+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/docker v20.10.17+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
@@ -80,27 +90,35 @@ require (
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/open-policy-agent/opa v0.47.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/prometheus/prometheus v0.35.0 // indirect
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect