Merge pull request #1468 from gruntwork-io/ova-eval-withoutput

Opa EvalWithOutput

modules/opa/eval.go

@@ -61,20 +61,52 @@ func Eval(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resu
 //	opa eval -i $JSONFile -d $RulePath $ResultQuery
 //
 // This will asynchronously run OPA on each file concurrently using goroutines.
-func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) error {
+// This will fail the test if any one of the files failed.
+// For each file, the output will be returned on the outputs slice.
+func EvalWithOutput(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string) {
+	outputs, err := EvalWithOutputE(t, options, jsonFilePaths, resultQuery)
+	require.NoError(t, err)
+	return
+}
+
+// EvalE runs `opa eval` on the given JSON files using the configured policy file and result query. Translates to:
+//
+//	opa eval -i $JSONFile -d $RulePath $ResultQuery
+//
+// This will asynchronously run OPA on each file concurrently using goroutines.
+func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (err error) {
+	_, err = evalE(t, options, jsonFilePaths, resultQuery)
+	return
+}
+
+// EvalWithOutputE runs `opa eval` on the given JSON files using the configured policy file and result query. Translates to:
+//
+//	opa eval -i $JSONFile -d $RulePath $ResultQuery
+//
+// This will asynchronously run OPA on each file concurrently using goroutines.
+// For each file, the output will be returned on the outputs slice.
+func EvalWithOutputE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string, err error) {
+	return evalE(t, options, jsonFilePaths, resultQuery)
+}
+
+func evalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string, err error) {
 	downloadedPolicyPath, err := DownloadPolicyE(t, options.RulePath)
 	if err != nil {
-		return err
+		return
 	}
 
+	outputs = make([]string, len(jsonFilePaths))
 	wg := new(sync.WaitGroup)
 	wg.Add(len(jsonFilePaths))
 	errorsOccurred := new(multierror.Error)
 	errChans := make([]chan error, len(jsonFilePaths))
 	for i, jsonFilePath := range jsonFilePaths {
 		errChan := make(chan error, 1)
 		errChans[i] = errChan
-		go asyncEval(t, wg, errChan, options, downloadedPolicyPath, jsonFilePath, resultQuery)
+
+		go func(i int, jsonFilePath string) {
+			outputs[i] = asyncEval(t, wg, errChan, options, downloadedPolicyPath, jsonFilePath, resultQuery)
+		}(i, jsonFilePath)
 	}
 	wg.Wait()
 	for _, errChan := range errChans {
@@ -83,7 +115,7 @@ func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, res
 			errorsOccurred = multierror.Append(errorsOccurred, err)
 		}
 	}
-	return errorsOccurred.ErrorOrNil()
+	return outputs, errorsOccurred.ErrorOrNil()
 }
 
 // asyncEval is a function designed to be run in a goroutine to asynchronously call `opa eval` on a single input file.
@@ -95,7 +127,7 @@ func asyncEval(
 	downloadedPolicyPath string,
 	jsonFilePath string,
 	resultQuery string,
-) {
+) (output string) {
 	defer wg.Done()
 	cmd := shell.Command{
 		Command: "opa",
@@ -105,7 +137,7 @@ func asyncEval(
 		// opa eval is typically very quick.
 		Logger: logger.Discard,
 	}
-	err := runCommandWithFullLoggingE(t, options.Logger, cmd)
+	output, err := runCommandWithFullLoggingE(t, options.Logger, cmd)
 	ruleBasePath := filepath.Base(downloadedPolicyPath)
 	if err == nil {
 		options.Logger.Logf(t, "opa eval passed on file %s (policy %s; query %s)", jsonFilePath, ruleBasePath, resultQuery)
@@ -115,10 +147,12 @@ func asyncEval(
 			options.Logger.Logf(t, "DEBUG: rerunning opa eval to query for full data.")
 			cmd.Args = formatOPAEvalArgs(options, downloadedPolicyPath, jsonFilePath, "data")
 			// We deliberately ignore the error here as we want to only return the original error.
-			runCommandWithFullLoggingE(t, options.Logger, cmd)
+			output, _ = runCommandWithFullLoggingE(t, options.Logger, cmd)
 		}
 	}
 	errChan <- err
+
+	return
 }
 
 // formatOPAEvalArgs formats the arguments for the `opa eval` command.
@@ -146,8 +180,8 @@ func formatOPAEvalArgs(options *EvalOptions, rulePath, jsonFilePath, resultQuery
 // runCommandWithFullLogging will log the command output in its entirety with buffering. This avoids breaking up the
 // logs when commands are run concurrently. This is a private function used in the context of opa only because opa runs
 // very quickly, and the output of opa is hard to parse if it is broken up by interleaved logs.
-func runCommandWithFullLoggingE(t testing.TestingT, logger *logger.Logger, cmd shell.Command) error {
-	output, err := shell.RunCommandAndGetOutputE(t, cmd)
+func runCommandWithFullLoggingE(t testing.TestingT, logger *logger.Logger, cmd shell.Command) (output string, err error) {
+	output, err = shell.RunCommandAndGetOutputE(t, cmd)
 	logger.Logf(t, "Output of command `%s %s`:\n%s", cmd.Command, strings.Join(cmd.Args, " "), output)
-	return err
+	return
 }

modules/opa/eval_test.go

@@ -0,0 +1,155 @@
+package opa
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEvalWithOutput(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+
+		policy  string
+		query   string
+		inputs  []string
+		outputs []string
+		isError bool
+	}{
+		{
+			name: "Success",
+			policy: `
+				package test
+				allow {
+					startswith(input.user, "admin")
+				}
+			`,
+			query: "data.test.allow",
+			inputs: []string{
+				`{"user": "admin-1"}`,
+				`{"user": "admin-2"}`,
+				`{"user": "admin-3"}`,
+			},
+			outputs: []string{
+				`{
+					"result": [{
+						"expressions": [{
+							"value": true,
+							"text": "data.test.allow",
+							"location": {
+								"row": 1,
+								"col": 1
+							}
+						}]
+					}]
+				}`,
+				`{
+					"result": [{
+						"expressions": [{
+							"value": true,
+							"text": "data.test.allow",
+							"location": {
+								"row": 1,
+								"col": 1
+							}
+						}]
+					}]
+				}`,
+				`{
+					"result": [{
+						"expressions": [{
+							"value": true,
+							"text": "data.test.allow",
+							"location": {
+								"row": 1,
+								"col": 1
+							}
+						}]
+					}]
+				}`,
+			},
+		},
+		{
+			name: "ContainsError",
+			policy: `
+				package test
+				allow {
+					input.user == "admin"
+				}
+			`,
+			query:   "data.test.allow",
+			isError: true,
+			inputs: []string{
+				`{"user": "admin"}`,
+				`{"user": "nobody"}`,
+			},
+			outputs: []string{
+				`{
+					"result": [{
+						"expressions": [{
+							"value": true,
+							"text": "data.test.allow",
+							"location": {
+								"row": 1,
+								"col": 1
+							}
+						}]
+					}]
+				}`,
+				`{
+					"result": [{
+						"expressions": [{
+							"value": {
+								"test": {}
+							},
+							"text": "data",
+							"location": {
+								"row": 1,
+								"col": 1
+							}
+						}]
+					}]
+				}`,
+			},
+		},
+	}
+
+	createTempFile := func(t *testing.T, name string, content string) string {
+		f, err := os.CreateTemp(t.TempDir(), name)
+		require.NoError(t, err)
+		t.Cleanup(func() { os.Remove(f.Name()) })
+		_, err = f.WriteString(content)
+		require.NoError(t, err)
+		return f.Name()
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			policy := createTempFile(t, "policy-*.rego", test.policy)
+			inputs := make([]string, len(test.inputs))
+			for i, input := range test.inputs {
+				f := createTempFile(t, "inputs-*.json", input)
+				inputs[i] = f
+			}
+
+			options := &EvalOptions{
+				RulePath: policy,
+			}
+
+			outputs, err := EvalWithOutputE(t, options, inputs, test.query)
+			if test.isError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			for i, output := range test.outputs {
+				require.JSONEq(t, output, outputs[i], "output for input: %d", i)
+			}
+		})
+	}
+}