Merge pull request cloudflare#548 from cloudflare/fail-on

Add pint ci --fail-on flag

cmd/pint/ci.go

@@ -22,6 +22,7 @@ import (
 var (
 	baseBranchFlag = "base-branch"
 	devFlag        = "dev"
+	failOnFlag     = "fail-on"
 )
 
 var ciCmd = &cli.Command{
@@ -47,6 +48,12 @@ var ciCmd = &cli.Command{
 			Value:   false,
 			Usage:   "Use experimental change detection",
 		},
+		&cli.StringFlag{
+			Name:    failOnFlag,
+			Aliases: []string{"w"},
+			Value:   "bug",
+			Usage:   "Exit with non-zero code if there are problems with given severity (or higher) detected",
+		},
 	},
 }
 
@@ -135,13 +142,14 @@ func actionCI(c *cli.Context) error {
 			return fmt.Errorf("GITHUB_PULL_REQUEST_NUMBER env variable is required when reporting to GitHub")
 		}
 
-		prNum, err := strconv.Atoi(prVal)
-		if err != nil {
+		var prNum int
+		if prNum, err = strconv.Atoi(prVal); err != nil {
 			return fmt.Errorf("got not a valid number via GITHUB_PULL_REQUEST_NUMBER: %w", err)
 		}
 
 		timeout, _ := time.ParseDuration(meta.cfg.Repository.GitHub.Timeout)
-		gr, err := reporter.NewGithubReporter(
+		var gr reporter.GithubReporter
+		if gr, err = reporter.NewGithubReporter(
 			version,
 			meta.cfg.Repository.GitHub.BaseURI,
 			meta.cfg.Repository.GitHub.UploadURI,
@@ -151,18 +159,22 @@ func actionCI(c *cli.Context) error {
 			meta.cfg.Repository.GitHub.Repo,
 			prNum,
 			git.RunGit,
-		)
-		if err != nil {
+		); err != nil {
 			return err
 		}
 		reps = append(reps, gr)
 	}
 
-	foundBugOrHigher := false
+	minSeverity, err := checks.ParseSeverity(c.String(failOnFlag))
+	if err != nil {
+		return fmt.Errorf("invalid --%s value: %w", failOnFlag, err)
+	}
+
+	problemsFound := false
 	bySeverity := map[string]interface{}{} // interface{} is needed for log.Fields()
 	for s, c := range summary.CountBySeverity() {
-		if s >= checks.Bug {
-			foundBugOrHigher = true
+		if s >= minSeverity {
+			problemsFound = true
 		}
 		bySeverity[s.String()] = c
 	}
@@ -174,7 +186,7 @@ func actionCI(c *cli.Context) error {
 		return fmt.Errorf("submitting reports: %w", err)
 	}
 
-	if foundBugOrHigher {
+	if problemsFound {
 		return fmt.Errorf("problems found")
 	}
 

cmd/pint/lint.go

@@ -67,7 +67,7 @@ func actionLint(c *cli.Context) error {
 
 	minSeverity, err := checks.ParseSeverity(c.String(minSeverityFlag))
 	if err != nil {
-		return fmt.Errorf("invalid %s value: %w", minSeverityFlag, err)
+		return fmt.Errorf("invalid --%s value: %w", minSeverityFlag, err)
 	}
 
 	r := reporter.NewConsoleReporter(os.Stderr, minSeverity)

cmd/pint/tests/0051_watch_severity_invalid.txt

@@ -3,4 +3,4 @@ pint.error --no-color watch --min-severity=foo bar
 cmp stderr stderr.txt
 
 -- stderr.txt --
-level=fatal msg="Fatal error" error="invalid min-severity value: unknown severity: foo"
+level=fatal msg="Fatal error" error="invalid --min-severity value: unknown severity: foo"

cmd/pint/tests/0091_lint_min_severity_invalid.txt

@@ -3,7 +3,7 @@ pint.error --no-color lint --min-severity=xxx rules
 cmp stderr stderr.txt
 
 -- stderr.txt --
-level=fatal msg="Fatal error" error="invalid min-severity value: unknown severity: xxx"
+level=fatal msg="Fatal error" error="invalid --min-severity value: unknown severity: xxx"
 -- rules/0001.yml --
 groups:
 - name: foo

cmd/pint/tests/0119_ci_fail_on_warning.txt

@@ -0,0 +1,65 @@
+http method github GET /api/v3/repos/cloudflare/pint/pulls/1/reviews 200 []
+http method github POST /api/v3/repos/cloudflare/pint/pulls/1/reviews 200 {}
+http method github GET /api/v3/repos/cloudflare/pint/pulls/1/comments 200 []
+http method github POST /api/v3/repos/cloudflare/pint/pulls/1/comments 200 {}
+http start github 127.0.0.1:6119
+
+mkdir testrepo
+cd testrepo
+exec git init --initial-branch=main .
+
+cp ../src/v1.yml rules.yml
+cp ../src/.pint.hcl .
+env GIT_AUTHOR_NAME=pint
+env GIT_AUTHOR_EMAIL=pint@example.com
+env GIT_COMMITTER_NAME=pint
+env GIT_COMMITTER_EMAIL=pint@example.com
+exec git add .
+exec git commit -am 'import rules and config'
+
+exec git checkout -b v2
+cp ../src/v2.yml rules.yml
+exec git commit -am 'v2'
+
+env GITHUB_AUTH_TOKEN=12345
+env GITHUB_PULL_REQUEST_NUMBER=1
+pint.error -l debug --offline --no-color ci --fail-on=warning
+! stdout .
+stderr 'level=info msg="Pull request review created" status="200 OK"'
+
+-- src/v1.yml --
+- alert: rule1
+  expr: sum(foo) by(job)
+- alert: rule2
+  expr: sum(foo) by(job)
+  for: 0s
+
+-- src/v2.yml --
+- alert: rule1
+  expr: sum(foo) by(job)
+  for: 0s
+- alert: rule2
+  expr: sum(foo) by(job)
+  for: 0s
+
+-- src/.pint.hcl --
+ci {
+  baseBranch = "main"
+}
+parser {
+  relaxed = [".*"]
+}
+repository {
+  github {
+    baseuri   = "http://127.0.0.1:6119"
+	  uploaduri = "http://127.0.0.1:6119"
+    owner     = "cloudflare"
+    repo      = "pint"
+  }
+}
+rule {
+  label "severity" {
+    value = "critical|warning|info"
+    required = true
+  }
+}

cmd/pint/tests/0120_ci_fail_on_invalid.txt

@@ -0,0 +1,53 @@
+mkdir testrepo
+cd testrepo
+exec git init --initial-branch=main .
+
+cp ../src/v1.yml rules.yml
+cp ../src/.pint.hcl .
+env GIT_AUTHOR_NAME=pint
+env GIT_AUTHOR_EMAIL=pint@example.com
+env GIT_COMMITTER_NAME=pint
+env GIT_COMMITTER_EMAIL=pint@example.com
+exec git add .
+exec git commit -am 'import rules and config'
+
+exec git checkout -b v2
+cp ../src/v2.yml rules.yml
+exec git commit -am 'v2'
+
+env GITHUB_AUTH_TOKEN=12345
+env GITHUB_PULL_REQUEST_NUMBER=1
+pint.error -l debug --offline --no-color ci --fail-on=bogus
+! stdout .
+stderr 'level=fatal msg="Fatal error" error="invalid --fail-on value: unknown severity: bogus"'
+
+-- src/v1.yml --
+- alert: rule1
+  expr: sum(foo) by(job)
+- alert: rule2
+  expr: sum(foo) by(job)
+  for: 0s
+
+-- src/v2.yml --
+- alert: rule1
+  expr: sum(foo) by(job)
+  for: 0s
+- alert: rule2
+  expr: sum(foo) by(job)
+  for: 0s
+
+-- src/.pint.hcl --
+ci {
+  baseBranch = "main"
+}
+parser {
+  relaxed = [".*"]
+}
+repository {
+  github {
+    baseuri   = "http://127.0.0.1:6120"
+	  uploaduri = "http://127.0.0.1:6120"
+    owner     = "cloudflare"
+    repo      = "pint"
+  }
+}

cmd/pint/watch.go

@@ -85,7 +85,7 @@ func actionWatch(c *cli.Context) error {
 
 	minSeverity, err := checks.ParseSeverity(c.String(minSeverityFlag))
 	if err != nil {
-		return fmt.Errorf("invalid %s value: %w", minSeverityFlag, err)
+		return fmt.Errorf("invalid --%s value: %w", minSeverityFlag, err)
 	}
 
 	pidfile := c.String(pidfileFlag)

docs/changelog.md

@@ -2,6 +2,10 @@
 
 ## v0.41.0
 
+### Added
+
+- Added `--fail-on` flag to `pint ci` command - #525.
+
 ### Changed
 
 - `pint lint` output will now include severity level as a text label - #524.