From a417d24bfeeb90cdde77cf697b22d5d47efb6cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Mon, 22 Jul 2024 14:07:16 +0200 Subject: [PATCH] Add JUnit output format (#929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sebastian Hoß --- README.md | 3 +- cmd/constants.go | 2 ++ cmd/lint.go | 27 +++++++++++++++ docs/cicd.md | 9 +++-- go.mod | 1 + go.sum | 3 ++ pkg/reporter/reporter.go | 59 +++++++++++++++++++++++++++++++++ pkg/reporter/reporter_test.go | 62 +++++++++++++++++++++++++++++++++++ 8 files changed, 163 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e0f15f7..38be400d 100644 --- a/README.md +++ b/README.md @@ -599,12 +599,13 @@ are: from the linter report - `sarif` - [SARIF](https://sarifweb.azurewebsites.net/) JSON output, for consumption by tools processing code analysis reports +- `junit` - JUnit XML output, e.g. for CI servers like GitLab that show these results in a merge request. ## OPA Check and Strict Mode Linting with Regal assumes syntactically correct Rego. If there are errors parsing any files during linting, the process is aborted and any parser errors are logged similarly to OPA. OPA itself provides a "linter" of sorts, -via the `opa check` comand and its `--strict` flag. This checks the provided Rego files not only for syntax errors, +via the `opa check` command and its `--strict` flag. This checks the provided Rego files not only for syntax errors, but also for OPA [strict mode](https://www.openpolicyagent.org/docs/latest/policy-language/#strict-mode) violations. > **Note** It is recommended to run `opa check --strict` as part of your policy build process, and address any violations diff --git a/cmd/constants.go b/cmd/constants.go index c59d8402..316ed55f 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -13,4 +13,6 @@ const ( formatFestive = "festive" // formatSarif is the SARIF format value for the --format flag in various commands. formatSarif = "sarif" + // formatJunit is the JUnit format value for the --format flag in various commands. + formatJunit = "junit" ) diff --git a/cmd/lint.go b/cmd/lint.go index 392ae044..b0ff50e5 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( "time" "github.com/fatih/color" + "github.com/jstemmer/go-junit-report/v2/junit" "github.com/spf13/cobra" "gopkg.in/yaml.v3" @@ -372,6 +374,8 @@ func getReporter(format string, outputWriter io.Writer) (reporter.Reporter, erro return reporter.NewFestiveReporter(outputWriter), nil case formatSarif: return reporter.NewSarifReporter(outputWriter), nil + case formatJunit: + return reporter.NewJUnitReporter(outputWriter), nil default: return nil, fmt.Errorf("unknown format %s", format) } @@ -406,6 +410,29 @@ func formatError(format string, err error) error { } return fmt.Errorf("%s", string(bs)) + } else if format == formatJunit { + testSuites := junit.Testsuites{ + Name: "regal", + } + testsuite := junit.Testsuite{ + Name: "lint", + } + testsuite.AddTestcase(junit.Testcase{ + Name: "Command execution failed", + Error: &junit.Result{ + Message: err.Error(), + }, + }) + testSuites.AddSuite(testsuite) + + buf := &bytes.Buffer{} + + err := testSuites.WriteXML(buf) + if err != nil { + return fmt.Errorf("failed to format errors for output: %w", err) + } + + return fmt.Errorf("%s", buf.String()) } return err diff --git a/docs/cicd.md b/docs/cicd.md index f34fce44..c01f7b00 100644 --- a/docs/cicd.md +++ b/docs/cicd.md @@ -41,9 +41,14 @@ regal_lint_policies: name: ghcr.io/styrainc/regal:latest entrypoint: ['/bin/sh', '-c'] script: - - regal lint ./policy + - regal lint ./policy --format junit > regal-results.xml + artifacts: + reports: + junit: regal-results.xml + when: always rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' ``` -The above will run Regal on the `policy` directory when a merge request is created or updated. +The above will run Regal on the `policy` directory when a merge request is created or updated and will show linting +violations as part of the merge request. diff --git a/go.mod b/go.mod index 55f7edba..9b67a4e7 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.6.0 + github.com/jstemmer/go-junit-report/v2 v2.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/open-policy-agent/opa v0.66.0 diff --git a/go.sum b/go.sum index 301c1f18..744eb087 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,7 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -90,6 +91,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9K github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= +github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index 5699c82a..cb295b83 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -7,9 +7,11 @@ import ( "fmt" "io" "os" + "sort" "strings" "github.com/fatih/color" + "github.com/jstemmer/go-junit-report/v2/junit" "github.com/olekukonko/tablewriter" "github.com/owenrumney/go-sarif/v2/sarif" @@ -53,6 +55,12 @@ type SarifReporter struct { out io.Writer } +// JUnitReporter reports violations in the JUnit XML format +// (https://github.com/junit-team/junit5/blob/main/platform-tests/src/test/resources/jenkins-junit.xsd). +type JUnitReporter struct { + out io.Writer +} + // NewPrettyReporter creates a new PrettyReporter. func NewPrettyReporter(out io.Writer) PrettyReporter { return PrettyReporter{out: out} @@ -83,6 +91,11 @@ func NewSarifReporter(out io.Writer) SarifReporter { return SarifReporter{out: out} } +// NewJUnitReporter creates a new JUnitReporter. +func NewJUnitReporter(out io.Writer) JUnitReporter { + return JUnitReporter{out: out} +} + // Publish prints a pretty report to the configured output. func (tr PrettyReporter) Publish(_ context.Context, r report.Report) error { table := buildPrettyViolationsTable(r.Violations) @@ -423,3 +436,49 @@ func getUniqueViolationURLs(violations []report.Violation) map[string]string { return urls } + +// Publish prints a JUnit XML report to the configured output. +func (tr JUnitReporter) Publish(_ context.Context, r report.Report) error { + testSuites := junit.Testsuites{ + Name: "regal", + } + + // group by file & sort by file + files := make([]string, 0) + violationsPerFile := map[string][]report.Violation{} + + for _, violation := range r.Violations { + files = append(files, violation.Location.File) + violationsPerFile[violation.Location.File] = append(violationsPerFile[violation.Location.File], violation) + } + + sort.Strings(files) + + for _, file := range files { + testsuite := junit.Testsuite{ + Name: file, + } + + for _, violation := range violationsPerFile[file] { + testsuite.AddTestcase(junit.Testcase{ + Name: fmt.Sprintf("%s/%s: %s", violation.Category, violation.Title, violation.Description), + Classname: violation.Location.String(), + Failure: &junit.Result{ + Message: fmt.Sprintf("%s. To learn more, see: %s", violation.Description, getDocumentationURL(violation)), + Type: violation.Level, + Data: fmt.Sprintf("Rule: %s\nDescription: %s\nCategory: %s\nLocation: %s\nText: %s\nDocumentation: %s", + violation.Title, + violation.Description, + violation.Category, + violation.Location.String(), + strings.TrimSpace(*violation.Location.Text), + getDocumentationURL(violation)), + }, + }) + } + + testSuites.AddSuite(testsuite) + } + + return testSuites.WriteXML(tr.out) +} diff --git a/pkg/reporter/reporter_test.go b/pkg/reporter/reporter_test.go index d275eef5..e16f42a5 100644 --- a/pkg/reporter/reporter_test.go +++ b/pkg/reporter/reporter_test.go @@ -595,3 +595,65 @@ func TestSarifReporterPublishNoViolations(t *testing.T) { t.Errorf("expected %s, got %s", expect, buf.String()) } } + +//nolint:lll // the expected output is unfortunately longer than the allowed max line length +func TestJUnitReporterPublish(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + sr := NewJUnitReporter(&buf) + + err := sr.Publish(context.Background(), rep) + if err != nil { + t.Fatal(err) + } + + expect := ` + + + + + + + + + + + +` + + if buf.String() != expect { + t.Errorf("expected \n%s, got \n%s", expect, buf.String()) + } +} + +func TestJUnitReporterPublishNoViolations(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + sr := NewJUnitReporter(&buf) + + err := sr.Publish(context.Background(), report.Report{}) + if err != nil { + t.Fatal(err) + } + + expect := ` +` + + if buf.String() != expect { + t.Errorf("expected \n%s, got \n%s", expect, buf.String()) + } +}