diff --git a/clients/githubrepo/roundtripper/rate_limit.go b/clients/githubrepo/roundtripper/rate_limit.go index 60878f0c316..9730616d1f1 100644 --- a/clients/githubrepo/roundtripper/rate_limit.go +++ b/clients/githubrepo/roundtripper/rate_limit.go @@ -20,6 +20,9 @@ import ( "strconv" "time" + "go.opencensus.io/stats" + + githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/log" ) @@ -44,6 +47,17 @@ func (gh *rateLimitTransport) RoundTrip(r *http.Request) (*http.Response, error) if err != nil { return nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("innerTransport.RoundTrip: %v", err)) } + + retryValue := resp.Header.Get("Retry-After") + if retryAfter, err := strconv.Atoi(retryValue); err == nil { // if NO error + stats.Record(r.Context(), githubstats.RetryAfter.M(int64(retryAfter))) + duration := time.Duration(retryAfter) * time.Second + gh.logger.Info(fmt.Sprintf("Retry-After header set. Waiting %s to retry...", duration)) + time.Sleep(duration) + gh.logger.Info("Retry-After header set. Retrying...") + return gh.RoundTrip(r) + } + rateLimit := resp.Header.Get("X-RateLimit-Remaining") remaining, err := strconv.Atoi(rateLimit) if err != nil { diff --git a/clients/githubrepo/roundtripper/transport.go b/clients/githubrepo/roundtripper/transport.go index a9824edc2fe..fa606979459 100644 --- a/clients/githubrepo/roundtripper/transport.go +++ b/clients/githubrepo/roundtripper/transport.go @@ -64,5 +64,6 @@ func (gt *githubTransport) RoundTrip(r *http.Request) (*http.Response, error) { if err == nil { stats.Record(ctx, githubstats.RemainingTokens.M(int64(remaining))) } + return resp, nil } diff --git a/clients/githubrepo/stats/stats.go b/clients/githubrepo/stats/stats.go index e320c910478..a6d8821a96a 100644 --- a/clients/githubrepo/stats/stats.go +++ b/clients/githubrepo/stats/stats.go @@ -24,7 +24,9 @@ var ( // RemainingTokens measures the remaining number of API tokens. RemainingTokens = stats.Int64("RemainingTokens", "Measures the remaining count of API tokens", stats.UnitDimensionless) - + // RetryAfter measures the retry delay when dealing with secondary rate limits. + RetryAfter = stats.Int64("RetryAfter", + "Measures the retry delay when dealing with secondary rate limits", stats.UnitSeconds) // TokenIndex is the tag key for specifying a unique token. TokenIndex = tag.MustNewKey("tokenIndex") // ResourceType specifies the type of GitHub resource. diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 27e7d2643f0..a9d4017f82a 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -43,7 +43,9 @@ additional-params: api-results-bucket-url: gs://ossf-scorecard-cron-results # TODO: Temporarily remove SAST and CI-Tests which require lot of GitHub API tokens. # TODO(#859): Re-add Contributors after fixing inconsistencies. - blacklisted-checks: CI-Tests,Contributors + # TODO: Dependency-Update-Tool, Fuzzing, and SAST are search heavy + # TODO: Vulnerabilities is resource intensive, wait until the next osv-scanner release after v1.2.0 + blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool,Fuzzing,SAST,Vulnerabilities cii-data-bucket-url: gs://ossf-scorecard-cii-data # Raw results. raw-bigquery-table: scorecard-rawdata diff --git a/cron/config/config_test.go b/cron/config/config_test.go index 82e607f3123..d11290cd149 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -34,7 +34,7 @@ const ( prodCompletionThreshold = 0.99 prodWebhookURL = "" prodCIIDataBucket = "gs://ossf-scorecard-cii-data" - prodBlacklistedChecks = "CI-Tests,Contributors" + prodBlacklistedChecks = "CI-Tests,Contributors,Dependency-Update-Tool,Fuzzing,SAST,Vulnerabilities" prodShardSize int = 10 prodMetricExporter string = "stackdriver" prodMetricStackdriverPrefix string = "scorecard-cron" diff --git a/cron/k8s/auth.yaml b/cron/k8s/auth.yaml index a1da9537e88..dbfa2c0b723 100644 --- a/cron/k8s/auth.yaml +++ b/cron/k8s/auth.yaml @@ -31,7 +31,7 @@ kind: Deployment metadata: name: scorecard-github-server spec: - replicas: 1 + replicas: 0 selector: matchLabels: app.kubernetes.io/name: github-auth-server diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index dc445052a41..be78ad1318b 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -29,7 +29,7 @@ spec: containers: - name: worker image: gcr.io/openssf/scorecard-batch-worker:latest - args: ["--ignoreRuntimeErrors=false", "--config=/etc/scorecard/config.yaml"] + args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"] imagePullPolicy: Always env: - name: SCORECARD_DATA_BUCKET_URL @@ -40,10 +40,22 @@ spec: value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest" - name: SCORECARD_METRIC_EXPORTER value: "printer" - - name: GITHUB_AUTH_SERVER - value: "10.4.4.210:80" + - name: GITHUB_APP_KEY_PATH + value: /etc/github/app_key + - name: GITHUB_APP_ID + valueFrom: + secretKeyRef: + name: github + key: app_id + - name: GITHUB_APP_INSTALLATION_ID + valueFrom: + secretKeyRef: + name: github + key: installation_id - name: "SCORECARD_API_RESULTS_BUCKET_URL" value: "gs://ossf-scorecard-cron-releasetest-results" + - name: "SCORECARD_BLACKLISTED_CHECKS" + value: "CI-Tests,Contributors,Dependency-Update-Tool,SAST" resources: requests: memory: 5Gi @@ -55,10 +67,16 @@ spec: - name: config-volume mountPath: /etc/scorecard readOnly: true + - name: github-app-key + mountPath: "/etc/github/" + readOnly: true volumes: - name: config-volume configMap: name: scorecard-config + - name: github-app-key + secret: + secretName: github strategy: type: "RollingUpdate" rollingUpdate: diff --git a/cron/k8s/worker.yaml b/cron/k8s/worker.yaml index 9ffe36809e1..9307d2d8d57 100644 --- a/cron/k8s/worker.yaml +++ b/cron/k8s/worker.yaml @@ -32,8 +32,18 @@ spec: args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"] imagePullPolicy: Always env: - - name: GITHUB_AUTH_SERVER - value: "10.4.4.210:80" + - name: GITHUB_APP_KEY_PATH + value: /etc/github/app_key + - name: GITHUB_APP_ID + valueFrom: + secretKeyRef: + name: github + key: app_id + - name: GITHUB_APP_INSTALLATION_ID + valueFrom: + secretKeyRef: + name: github + key: installation_id resources: requests: memory: 5Gi @@ -45,10 +55,16 @@ spec: - name: config-volume mountPath: /etc/scorecard readOnly: true + - name: github-app-key + mountPath: "/etc/github/" + readOnly: true volumes: - name: config-volume configMap: name: scorecard-config + - name: github-app-key + secret: + secretName: github strategy: type: "RollingUpdate" rollingUpdate: