add allow_net to capabilities, use it to disable fetching remote sc…

…hemas (#3748)

This adds a new top-level key to the capabilities structure, `allow_net`.
It currently is only used for restricting the typechecker's ability to fetch
remote refs in JSON schemas, but could be used more widely in the future.

It works like this:

- If it's not present, any host can be contacted
- If it's present, the items will be the hosts or IP addresses that may be
   contacted; anything not in the list is prohibited.
- As a consequence, If it's present and empty (`[]`), no host can be contacted

Introducing a package-level var to gojsonschema isn't the prettiest solution,
but since we want this in an all-or-nothing way right now anyways, it does
the trick. And it's more ergonomic than adding extra parameters all over the
place.

Fixes #3746.

Also:

* move some profiling-related default params into newEvalCommandParams
* replace some errors.Wrap by fmt.Errorf in loader pkg
* remove some != nil handling where it didn't make a difference when
  working on the schema set
* reduces indentation in code examples in `opa eval -h` and `opa check -h`
  by replacing tabs by four spaces.
* ast: allow testing with remote refs without networking

It would be nice to ensure that the remote refs feature actually works,
without introducing a network dependency into our tests.

This commit adds the kube 1.14 definitions into ast/testdata, and uses
that from a httptest.Server instance in the unit tests.

Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>

ast/capabilities.go

@@ -15,8 +15,18 @@ import (
 // Capabilities defines a structure containing data that describes the capablilities
 // or features supported by a particular version of OPA.
 type Capabilities struct {
-	Builtins        []*Builtin       `json:"builtins"` // builtins is a set of built-in functions that are supported.
+	// builtins is a set of built-in functions that are supported.
+	Builtins        []*Builtin       `json:"builtins"`
 	WasmABIVersions []WasmABIVersion `json:"wasm_abi_versions"`
+
+	// allow_net is an array of hostnames or IP addresses, that an OPA instance is
+	// allowed to connect to.
+	// If omitted, ANY host can be connected to. If empty, NO host can be connected to.
+	// As of now, this only controls fetching remote refs for using JSON Schemas in
+	// the type checker.
+	// TODO(sr): support ports to further restrict connection peers
+	// TODO(sr): support restricting `http.send` using the same mechanism (see https://github.com/open-policy-agent/opa/issues/3665)
+	AllowNet []string `json:"allow_net,omitempty"`
 }
 
 // WasmABIVersion captures the Wasm ABI version. Its `Minor` version is indicating

ast/check.go

@@ -28,6 +28,7 @@ type typeChecker struct {
 	exprCheckers map[string]exprChecker
 	varRewriter  rewriteVars
 	ss           *SchemaSet
+	allowNet     []string
 	input        types.Type
 }
 
@@ -55,6 +56,7 @@ func (tc *typeChecker) copy() *typeChecker {
 	return newTypeChecker().
 		WithVarRewriter(tc.varRewriter).
 		WithSchemaSet(tc.ss).
+		WithAllowNet(tc.allowNet).
 		WithInputType(tc.input)
 }
 
@@ -63,6 +65,11 @@ func (tc *typeChecker) WithSchemaSet(ss *SchemaSet) *typeChecker {
 	return tc
 }
 
+func (tc *typeChecker) WithAllowNet(hosts []string) *typeChecker {
+	tc.allowNet = hosts
+	return tc
+}
+
 func (tc *typeChecker) WithVarRewriter(f rewriteVars) *typeChecker {
 	tc.varRewriter = f
 	return tc
@@ -180,7 +187,7 @@ func (tc *typeChecker) checkRule(env *TypeEnv, as *annotationSet, rule *Rule) {
 
 	if schemaAnnots := getRuleAnnotation(as, rule); schemaAnnots != nil {
 		for _, schemaAnnot := range schemaAnnots {
-			ref, refType, err := processAnnotation(tc.ss, schemaAnnot, rule)
+			ref, refType, err := processAnnotation(tc.ss, schemaAnnot, rule, tc.allowNet)
 			if err != nil {
 				tc.err([]*Error{err})
 				continue
@@ -1178,7 +1185,7 @@ func getRuleAnnotation(as *annotationSet, rule *Rule) (result []*SchemaAnnotatio
 	return result
 }
 
-func processAnnotation(ss *SchemaSet, annot *SchemaAnnotation, rule *Rule) (Ref, types.Type, *Error) {
+func processAnnotation(ss *SchemaSet, annot *SchemaAnnotation, rule *Rule, allowNet []string) (Ref, types.Type, *Error) {
 
 	var schema interface{}
 
@@ -1191,7 +1198,7 @@ func processAnnotation(ss *SchemaSet, annot *SchemaAnnotation, rule *Rule) (Ref,
 		schema = *annot.Definition
 	}
 
-	tpe, err := loadSchema(schema)
+	tpe, err := loadSchema(schema, allowNet)
 	if err != nil {
 		return nil, nil, NewError(TypeErr, rule.Location, err.Error())
 	}

ast/compile.go

@@ -867,7 +867,9 @@ func (c *Compiler) checkSafetyRuleHeads() {
 	}
 }
 
-func compileSchema(goSchema interface{}) (*gojsonschema.Schema, error) {
+func compileSchema(goSchema interface{}, allowNet []string) (*gojsonschema.Schema, error) {
+	gojsonschema.SetAllowNet(allowNet)
+
 	var refLoader gojsonschema.JSONLoader
 	sl := gojsonschema.NewSchemaLoader()
 
@@ -878,7 +880,7 @@ func compileSchema(goSchema interface{}) (*gojsonschema.Schema, error) {
 	}
 	schemasCompiled, err := sl.Compile(refLoader)
 	if err != nil {
-		return nil, fmt.Errorf("unable to compile the schema due to: %w", err)
+		return nil, fmt.Errorf("unable to compile the schema: %w", err)
 	}
 	return schemasCompiled, nil
 }
@@ -1139,7 +1141,7 @@ func (c *Compiler) init() {
 	// Load the global input schema if one was provided.
 	if c.schemaSet != nil {
 		if schema := c.schemaSet.Get(SchemaRootRef); schema != nil {
-			tpe, err := loadSchema(schema)
+			tpe, err := loadSchema(schema, c.capabilities.AllowNet)
 			if err != nil {
 				c.err(NewError(TypeErr, nil, err.Error()))
 			} else {

ast/schema.go

@@ -47,16 +47,16 @@ func (ss *SchemaSet) Get(path Ref) interface{} {
 	return x
 }
 
-func loadSchema(raw interface{}) (types.Type, error) {
+func loadSchema(raw interface{}, allowNet []string) (types.Type, error) {
 
-	jsonSchema, err := compileSchema(raw)
+	jsonSchema, err := compileSchema(raw, allowNet)
 	if err != nil {
-		return nil, fmt.Errorf("compile failed: %s", err.Error())
+		return nil, err
 	}
 
 	tpe, err := parseSchema(jsonSchema.RootSchema)
 	if err != nil {
-		return nil, fmt.Errorf("error when type checking %v", err)
+		return nil, fmt.Errorf("type checking: %w", err)
 	}
 
 	return tpe, nil

ast/schema_test.go

@@ -3,6 +3,10 @@ package ast
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/open-policy-agent/opa/types"
@@ -17,7 +21,7 @@ func testParseSchema(t *testing.T, schema string, expectedType types.Type, expec
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	newtype, err := loadSchema(sch)
+	newtype, err := loadSchema(sch, nil)
 	if err != nil && errors.Is(err, expectedError) {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -49,29 +53,76 @@ func TestParseSchemaObject(t *testing.T) {
 
 func TestSetTypesWithSchemaRef(t *testing.T) {
 	var sch interface{}
-	err := util.Unmarshal([]byte(refSchema), &sch)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	newtype, err := loadSchema(sch)
+
+	ts := kubeSchemaServer(t)
+	t.Cleanup(ts.Close)
+	refSchemaReplaced := strings.ReplaceAll(refSchema, "https://kubernetesjsonschema.dev/v1.14.0/", ts.URL+"/")
+	err := util.Unmarshal([]byte(refSchemaReplaced), &sch)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if newtype == nil {
-		t.Fatalf("parseSchema returned nil type")
-	}
-	if newtype.String() != "object<apiVersion: string, kind: string, metadata: object<annotations: object[any: any], clusterName: string, creationTimestamp: string, deletionGracePeriodSeconds: number, deletionTimestamp: string, finalizers: array[string], generateName: string, generation: number, initializers: object<pending: array[object<name: string>], result: object<apiVersion: string, code: number, details: object<causes: array[object<field: string, message: string, reason: string>], group: string, kind: string, name: string, retryAfterSeconds: number, uid: string>, kind: string, message: string, metadata: object<continue: string, resourceVersion: string, selfLink: string>, reason: string, status: string>>, labels: object[any: any], managedFields: array[object<apiVersion: string, fields: object[any: any], manager: string, operation: string, time: string>], name: string, namespace: string, ownerReferences: array[object<apiVersion: string, blockOwnerDeletion: boolean, controller: boolean, kind: string, name: string, uid: string>], resourceVersion: string, selfLink: string, uid: string>>" {
-		t.Fatalf("parseSchema returned an incorrect type: %s", newtype.String())
-	}
+
+	t.Run("remote refs disabled", func(t *testing.T) {
+		_, err := loadSchema(sch, []string{})
+		if err == nil {
+			t.Fatal("expected error, got nil")
+		}
+		expErr := fmt.Sprintf("unable to compile the schema: remote reference loading disabled: %s/_definitions.json", ts.URL)
+		if exp, act := expErr, err.Error(); act != exp {
+			t.Errorf("expected message %q, got %q", exp, act)
+		}
+	})
+
+	t.Run("all remote refs enabled", func(t *testing.T) {
+		newtype, err := loadSchema(sch, nil)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if newtype == nil {
+			t.Fatalf("parseSchema returned nil type")
+		}
+		if newtype.String() != "object<apiVersion: string, kind: string, metadata: object<annotations: object[any: any], clusterName: string, creationTimestamp: string, deletionGracePeriodSeconds: number, deletionTimestamp: string, finalizers: array[string], generateName: string, generation: number, initializers: object<pending: array[object<name: string>], result: object<apiVersion: string, code: number, details: object<causes: array[object<field: string, message: string, reason: string>], group: string, kind: string, name: string, retryAfterSeconds: number, uid: string>, kind: string, message: string, metadata: object<continue: string, resourceVersion: string, selfLink: string>, reason: string, status: string>>, labels: object[any: any], managedFields: array[object<apiVersion: string, fields: object[any: any], manager: string, operation: string, time: string>], name: string, namespace: string, ownerReferences: array[object<apiVersion: string, blockOwnerDeletion: boolean, controller: boolean, kind: string, name: string, uid: string>], resourceVersion: string, selfLink: string, uid: string>>" {
+			t.Fatalf("parseSchema returned an incorrect type: %s", newtype.String())
+		}
+	})
+
+	t.Run("desired remote ref selectively enabled", func(t *testing.T) {
+		newtype, err := loadSchema(sch, []string{"127.0.0.1"})
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if newtype == nil {
+			t.Fatalf("parseSchema returned nil type")
+		}
+		if newtype.String() != "object<apiVersion: string, kind: string, metadata: object<annotations: object[any: any], clusterName: string, creationTimestamp: string, deletionGracePeriodSeconds: number, deletionTimestamp: string, finalizers: array[string], generateName: string, generation: number, initializers: object<pending: array[object<name: string>], result: object<apiVersion: string, code: number, details: object<causes: array[object<field: string, message: string, reason: string>], group: string, kind: string, name: string, retryAfterSeconds: number, uid: string>, kind: string, message: string, metadata: object<continue: string, resourceVersion: string, selfLink: string>, reason: string, status: string>>, labels: object[any: any], managedFields: array[object<apiVersion: string, fields: object[any: any], manager: string, operation: string, time: string>], name: string, namespace: string, ownerReferences: array[object<apiVersion: string, blockOwnerDeletion: boolean, controller: boolean, kind: string, name: string, uid: string>], resourceVersion: string, selfLink: string, uid: string>>" {
+			t.Fatalf("parseSchema returned an incorrect type: %s", newtype.String())
+		}
+	})
+
+	t.Run("different remote ref selectively enabled", func(t *testing.T) {
+		_, err := loadSchema(sch, []string{"foo"})
+		if err == nil {
+			t.Fatal("expected error, got nil")
+		}
+		expErr := fmt.Sprintf("unable to compile the schema: remote reference loading disabled: %s/_definitions.json", ts.URL)
+		if exp, act := expErr, err.Error(); act != exp {
+			t.Errorf("expected message %q, got %q", exp, act)
+		}
+	})
 }
 
 func TestSetTypesWithPodSchema(t *testing.T) {
 	var sch interface{}
-	err := util.Unmarshal([]byte(podSchema), &sch)
+
+	ts := kubeSchemaServer(t)
+	t.Cleanup(ts.Close)
+
+	podSchemaReplaced := strings.ReplaceAll(podSchema, "https://kubernetesjsonschema.dev/v1.14.0/", ts.URL+"/")
+	err := util.Unmarshal([]byte(podSchemaReplaced), &sch)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	newtype, err := loadSchema(sch)
+	newtype, err := loadSchema(sch, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -389,7 +440,7 @@ func TestCompileSchemaEmptySchema(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	jsonSchema, _ := compileSchema(sch)
+	jsonSchema, _ := compileSchema(sch, []string{})
 	if jsonSchema != nil {
 		t.Fatalf("Incorrect return from parseSchema with an empty schema")
 	}
@@ -401,7 +452,7 @@ func TestParseSchemaWithSchemaBadSchema(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	jsonSchema, err := compileSchema(sch)
+	jsonSchema, err := compileSchema(sch, []string{})
 	if err != nil {
 		t.Fatalf("Unable to compile schema: %v", err)
 	}
@@ -508,3 +559,18 @@ func TestAnyOfSchema(t *testing.T) {
 		})
 	}
 }
+
+func kubeSchemaServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	bs, err := ioutil.ReadFile("testdata/_definitions.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, err := w.Write(bs)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	return ts
+}