🌱 Setup cron for running as GitHub App (#2721)

* Update auth server to use GitHub App.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Update release worker to use GitHub App tokens directly, as a workaround for the auth server not supporting it.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Add Retry-After logic and stats.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Change retry-after logic to support any status code. Disable troublesome checks.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Use GitHub App Token instead of auth server.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Temporarily disable additional chhecks.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Disable github auth server as it doesn't work with the GitHub App Tokens.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Re-enable Fuzzing check in the release test.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Fix unit test for new check change.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Move opencensus stat to the ratelimit roundtripped.

Signed-off-by: Spencer Schrock <sschrock@google.com>

---------

Signed-off-by: Spencer Schrock <sschrock@google.com>

clients/githubrepo/roundtripper/rate_limit.go

@@ -20,6 +20,9 @@ import (
 	"strconv"
 	"time"
 
+	"go.opencensus.io/stats"
+
+	githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/log"
 )
@@ -44,6 +47,17 @@ func (gh *rateLimitTransport) RoundTrip(r *http.Request) (*http.Response, error)
 	if err != nil {
 		return nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("innerTransport.RoundTrip: %v", err))
 	}
+
+	retryValue := resp.Header.Get("Retry-After")
+	if retryAfter, err := strconv.Atoi(retryValue); err == nil { // if NO error
+		stats.Record(r.Context(), githubstats.RetryAfter.M(int64(retryAfter)))
+		duration := time.Duration(retryAfter) * time.Second
+		gh.logger.Info(fmt.Sprintf("Retry-After header set. Waiting %s to retry...", duration))
+		time.Sleep(duration)
+		gh.logger.Info("Retry-After header set. Retrying...")
+		return gh.RoundTrip(r)
+	}
+
 	rateLimit := resp.Header.Get("X-RateLimit-Remaining")
 	remaining, err := strconv.Atoi(rateLimit)
 	if err != nil {

clients/githubrepo/roundtripper/transport.go

@@ -64,5 +64,6 @@ func (gt *githubTransport) RoundTrip(r *http.Request) (*http.Response, error) {
 	if err == nil {
 		stats.Record(ctx, githubstats.RemainingTokens.M(int64(remaining)))
 	}
+
 	return resp, nil
 }

clients/githubrepo/stats/stats.go

@@ -24,7 +24,9 @@ var (
 	// RemainingTokens measures the remaining number of API tokens.
 	RemainingTokens = stats.Int64("RemainingTokens",
 		"Measures the remaining count of API tokens", stats.UnitDimensionless)
-
+	// RetryAfter measures the retry delay when dealing with secondary rate limits.
+	RetryAfter = stats.Int64("RetryAfter",
+		"Measures the retry delay when dealing with secondary rate limits", stats.UnitSeconds)
 	// TokenIndex is the tag key for specifying a unique token.
 	TokenIndex = tag.MustNewKey("tokenIndex")
 	// ResourceType specifies the type of GitHub resource.

cron/config/config.yaml

@@ -43,7 +43,9 @@ additional-params:
     api-results-bucket-url: gs://ossf-scorecard-cron-results
     # TODO: Temporarily remove SAST and CI-Tests which require lot of GitHub API tokens.
     # TODO(#859): Re-add Contributors after fixing inconsistencies.
-    blacklisted-checks: CI-Tests,Contributors
+    # TODO: Dependency-Update-Tool, Fuzzing, and SAST are search heavy
+    # TODO: Vulnerabilities is resource intensive, wait until the next osv-scanner release after v1.2.0
+    blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool,Fuzzing,SAST,Vulnerabilities
     cii-data-bucket-url: gs://ossf-scorecard-cii-data
     # Raw results.
     raw-bigquery-table: scorecard-rawdata

cron/config/config_test.go

@@ -34,7 +34,7 @@ const (
 	prodCompletionThreshold            = 0.99
 	prodWebhookURL                     = ""
 	prodCIIDataBucket                  = "gs://ossf-scorecard-cii-data"
-	prodBlacklistedChecks              = "CI-Tests,Contributors"
+	prodBlacklistedChecks              = "CI-Tests,Contributors,Dependency-Update-Tool,Fuzzing,SAST,Vulnerabilities"
 	prodShardSize               int    = 10
 	prodMetricExporter          string = "stackdriver"
 	prodMetricStackdriverPrefix string = "scorecard-cron"

cron/k8s/auth.yaml

@@ -31,7 +31,7 @@ kind: Deployment
 metadata:
   name: scorecard-github-server
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       app.kubernetes.io/name: github-auth-server

cron/k8s/worker.release.yaml

@@ -29,7 +29,7 @@ spec:
       containers:
         - name: worker
           image: gcr.io/openssf/scorecard-batch-worker:latest
-          args: ["--ignoreRuntimeErrors=false", "--config=/etc/scorecard/config.yaml"]
+          args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"]
           imagePullPolicy: Always
           env:
             - name: SCORECARD_DATA_BUCKET_URL
@@ -40,10 +40,22 @@ spec:
               value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
             - name: SCORECARD_METRIC_EXPORTER
               value: "printer"
-            - name: GITHUB_AUTH_SERVER
-              value: "10.4.4.210:80"
+            - name: GITHUB_APP_KEY_PATH
+              value: /etc/github/app_key
+            - name: GITHUB_APP_ID
+              valueFrom:
+                secretKeyRef:
+                  name: github
+                  key: app_id
+            - name: GITHUB_APP_INSTALLATION_ID
+              valueFrom:
+                secretKeyRef:
+                  name: github
+                  key: installation_id
             - name: "SCORECARD_API_RESULTS_BUCKET_URL"
               value: "gs://ossf-scorecard-cron-releasetest-results"
+            - name: "SCORECARD_BLACKLISTED_CHECKS"
+              value: "CI-Tests,Contributors,Dependency-Update-Tool,SAST"
           resources:
             requests:
               memory: 5Gi
@@ -55,10 +67,16 @@ spec:
             - name: config-volume
               mountPath: /etc/scorecard
               readOnly: true
+            - name: github-app-key
+              mountPath: "/etc/github/"
+              readOnly: true
       volumes:
         - name: config-volume
           configMap:
             name: scorecard-config
+        - name: github-app-key
+          secret:
+            secretName: github
   strategy:
     type: "RollingUpdate"
     rollingUpdate:

cron/k8s/worker.yaml

@@ -32,8 +32,18 @@ spec:
           args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"]
           imagePullPolicy: Always
           env:
-            - name: GITHUB_AUTH_SERVER
-              value: "10.4.4.210:80"
+            - name: GITHUB_APP_KEY_PATH
+              value: /etc/github/app_key
+            - name: GITHUB_APP_ID
+              valueFrom:
+                secretKeyRef:
+                  name: github
+                  key: app_id
+            - name: GITHUB_APP_INSTALLATION_ID
+              valueFrom:
+                secretKeyRef:
+                  name: github
+                  key: installation_id
           resources:
             requests:
               memory: 5Gi
@@ -45,10 +55,16 @@ spec:
             - name: config-volume
               mountPath: /etc/scorecard
               readOnly: true
+            - name: github-app-key
+              mountPath: "/etc/github/"
+              readOnly: true
       volumes:
         - name: config-volume
           configMap:
             name: scorecard-config
+        - name: github-app-key
+          secret:
+            secretName: github
   strategy:
     type: "RollingUpdate"
     rollingUpdate: