Merge branch 'main' into feat/tools

Makefile

@@ -1,8 +1,8 @@
 SHELL := /bin/bash
 GOPATH := $(go env GOPATH)
-GINKGO := $(GOBIN)/ginkgo
-GOLANGGCI_LINT := $(GOBIN)/golangci-lint
-PROTOC_GEN_GO := $(GOBIN)/protoc-gen-go
+GINKGO := ginkgo
+GOLANGGCI_LINT := golangci-lint
+PROTOC_GEN_GO := protoc-gen-go
 PROTOC := $(shell which protoc)
 IMAGE_NAME = scorecard
 OUTPUT = output
@@ -80,7 +80,7 @@ build: $(build-targets)
 
 build-proto: ## Compiles and generates all required protobufs
 build-proto: cron/data/request.pb.go
-cron/data/request.pb.go: cron/data/request.proto | $(PROTOC_GEN_GO) $(PROTOC)
+cron/data/request.pb.go: cron/data/request.proto |  $(PROTOC)
 	protoc --go_out=../../../ cron/data/request.proto
 
 generate-docs: ## Generates docs
@@ -150,16 +150,17 @@ test-disk-cache: build-scorecard | $(GINKGO)
 	mkdir cache
 	@echo Focusing on these tests $(FOCUS_DISK_TEST)
 	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" \
-				   ./scorecard \
-				   --repo=https://github.com/ossf/scorecard \
-				   --show-details --metadata=openssf  --format json > ./$(OUTPUT)/results.json
-	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" $(GINKGO)  -p  -v -cover --focus=$(FOCUS_DISK_TEST)  ./e2e/...
+				./scorecard \
+				--repo=https://github.com/ossf/scorecard \
+				--show-details --metadata=openssf  --format json > ./$(OUTPUT)/results.json
+	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" ginkgo -p  -v -cover --focus=$(FOCUS_DISK_TEST)  ./e2e/...
 	# Rerun the same test with the disk cache filled to make sure the cache is working.
 	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" \
-				   ./scorecard \
-				   --repo=https://github.com/ossf/scorecard --show-details \
-				   --metadata=openssf  --format json > ./$(OUTPUT)/results.json
-	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" $(GINKGO)  -p  -v -cover --focus=$(FOCUS_DISK_TEST)  ./e2e/...
+		       		./scorecard \
+				--repo=https://github.com/ossf/scorecard --show-details \
+				--metadata=openssf  --format json > ./$(OUTPUT)/results.json
+	USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" ginkgo -p  -v -cover --focus=$(FOCUS_DISK_TEST)  ./e2e/...
+
 
 e2e-cron: ## Runs a e2e test cron job and validates its functionality
 	# Validate cron

cloudbuild/README.md

@@ -0,0 +1,3 @@
+# cloud builds
+
+The container images for scorecard and the cron job is built using `cloud build` in `gcp`.

cloudbuild/cron.yaml

@@ -0,0 +1,38 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build:
+  images:
+  - gcr.io/openssf/cron:latest
+  options:
+    diskSizeGb: '200'
+  steps:
+  - args:
+    - build
+    - -t
+    - gcr.io/openssf/cron:latest
+    - -f
+    - ./cron/Dockerfile
+    - .
+    name: gcr.io/cloud-builders/docker
+description: builds image for scorecard cron
+github:
+  name: scorecard
+  owner: ossf
+  push:
+    branch: ^main$
+name: scorecard-cron
+tags:
+- cron
+- scorecard

cloudbuild/scorecard-tag.yaml

@@ -0,0 +1,34 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build:
+  images:
+  - gcr.io/openssf/scorecard:$TAG_NAME
+  options: {}
+  steps:
+  - args:
+    - build
+    - -t
+    - gcr.io/openssf/scorecard:$TAG_NAME
+    - .
+    name: gcr.io/cloud-builders/docker
+description: scorecard build based on tag
+github:
+  name: scorecard
+  owner: ossf
+  push:
+    tag: .*
+name: scorecard-tag
+tags:
+- v*

cloudbuild/scorecard.yaml

@@ -0,0 +1,37 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build:
+  images:
+  - gcr.io/openssf/scorecard:latest
+  options:
+    diskSizeGb: '200'
+  steps:
+  - args:
+    - build
+    - -t
+    - gcr.io/openssf/scorecard:latest
+    - -f
+    - Dockerfile
+    - .
+    name: gcr.io/cloud-builders/docker
+description: 'builds image for scorecard '
+github:
+  name: scorecard
+  owner: ossf
+  push:
+    branch: ^main$
+name: scorecard
+tags:
+- scorecard

cmd/root.go

@@ -28,10 +28,13 @@ import (
 
 	goflag "flag"
 
+	"github.com/google/go-github/v32/github"
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/checks"
 	"github.com/ossf/scorecard/pkg"
 	"github.com/ossf/scorecard/repos"
+	"github.com/ossf/scorecard/roundtripper"
+	"github.com/shurcooL/githubv4"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -123,7 +126,13 @@ or ./scorecard --{npm,pypi,rubgems}=<package_name> [--checks=check1,...] [--show
 		}
 		ctx := context.Background()
 
-		repoResult := pkg.RunScorecards(ctx, sugar, repo, enabledChecks)
+		rt := roundtripper.NewTransport(ctx, sugar)
+		httpClient := &http.Client{
+			Transport: rt,
+		}
+		githubClient := github.NewClient(httpClient)
+		graphClient := githubv4.NewClient(httpClient)
+		repoResult := pkg.RunScorecards(ctx, repo, enabledChecks, httpClient, githubClient, graphClient)
 		repoResult.Metadata = append(repoResult.Metadata, metaData...)
 
 		// Sort them by name

cmd/serve.go

@@ -22,9 +22,12 @@ import (
 	"os"
 	"strings"
 
+	"github.com/google/go-github/v32/github"
 	"github.com/ossf/scorecard/checks"
 	"github.com/ossf/scorecard/pkg"
 	"github.com/ossf/scorecard/repos"
+	"github.com/ossf/scorecard/roundtripper"
+	"github.com/shurcooL/githubv4"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -65,7 +68,13 @@ var serveCmd = &cobra.Command{
 			}
 			sugar.Info(repoParam)
 			ctx := r.Context()
-			repoResult := pkg.RunScorecards(ctx, sugar, repo, checks.AllChecks)
+			rt := roundtripper.NewTransport(ctx, sugar)
+			httpClient := &http.Client{
+				Transport: rt,
+			}
+			githubClient := github.NewClient(httpClient)
+			graphClient := githubv4.NewClient(httpClient)
+			repoResult := pkg.RunScorecards(ctx, repo, checks.AllChecks, httpClient, githubClient, graphClient)
 
 			if r.Header.Get("Content-Type") == "application/json" {
 				if err := repoResult.AsJSON(showDetails, rw); err != nil {

cron/main.go

@@ -19,14 +19,18 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"time"
 
+	"github.com/google/go-github/v32/github"
 	"github.com/jszwec/csvutil"
 	"github.com/ossf/scorecard/checks"
 	"github.com/ossf/scorecard/pkg"
 	"github.com/ossf/scorecard/repos"
+	"github.com/ossf/scorecard/roundtripper"
+	"github.com/shurcooL/githubv4"
 	"go.uber.org/zap"
 )
 
@@ -60,6 +64,23 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+
+	ctx := context.Background()
+	cfg := zap.NewProductionConfig()
+	cfg.Level.SetLevel(zap.InfoLevel)
+	logger, err := cfg.Build()
+	if err != nil {
+		panic(err)
+	}
+	sugar := logger.Sugar()
+	// Use our custom roundtripper
+	rt := roundtripper.NewTransport(ctx, sugar)
+	httpClient := &http.Client{
+		Transport: rt,
+	}
+	githubClient := github.NewClient(httpClient)
+	graphClient := githubv4.NewClient(httpClient)
+
 	for _, r := range inputRepos {
 		fmt.Println(r.Repo)
 
@@ -71,21 +92,14 @@ func main() {
 			panic(err)
 		}
 
-		ctx := context.Background()
-		cfg := zap.NewProductionConfig()
-		cfg.Level.SetLevel(zap.InfoLevel)
-		logger, err := cfg.Build()
-		if err != nil {
-			panic(err)
-		}
-		sugar := logger.Sugar()
 		//nolint
 		// FIXME :- deleting branch-protection
 		// The branch protection check needs an admin access to the repository.
 		// All of the checks from cron would fail and uses another call to the API.
 		// This will reduce usage of the API.
 		delete(checks.AllChecks, "Branch-Protection")
-		repoResult := pkg.RunScorecards(ctx, sugar, repoURL, checks.AllChecks)
+
+		repoResult := pkg.RunScorecards(ctx, repoURL, checks.AllChecks, httpClient, githubClient, graphClient)
 		if err := repoResult.AsJSON( /*showDetails=*/ true, result); err != nil {
 			panic(err)
 		}

pkg/scorecard.go

@@ -23,52 +23,49 @@ import (
 	"github.com/google/go-github/v32/github"
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/repos"
-	"github.com/ossf/scorecard/roundtripper"
 	"github.com/shurcooL/githubv4"
-	"go.uber.org/zap"
 )
 
-func runEnabledChecks(ctx context.Context, logger *zap.SugaredLogger, repo repos.RepoURL,
-	checksToRun checker.CheckNameToFnMap, resultsCh chan checker.CheckResult) {
+func runEnabledChecks(ctx context.Context,
+	repo repos.RepoURL, checksToRun checker.CheckNameToFnMap,
+	httpClient *http.Client, githubClient *github.Client, graphClient *githubv4.Client,
+	resultsCh chan checker.CheckResult) {
+	request := checker.CheckRequest{
+		Ctx:         ctx,
+		Client:      githubClient,
+		HTTPClient:  httpClient,
+		Owner:       repo.Owner,
+		Repo:        repo.Repo,
+		GraphClient: graphClient,
+	}
 	wg := sync.WaitGroup{}
 	for _, checkFn := range checksToRun {
 		checkFn := checkFn
 		wg.Add(1)
 		go func() {
-			// Use our custom roundtripper
-			rt := roundtripper.NewTransport(ctx, logger)
-
-			client := &http.Client{
-				Transport: rt,
-			}
-			ghClient := github.NewClient(client)
-			graphClient := githubv4.NewClient(client)
-
-			c := checker.CheckRequest{
-				Ctx:         ctx,
-				Client:      ghClient,
-				HTTPClient:  client,
-				Owner:       repo.Owner,
-				Repo:        repo.Repo,
-				GraphClient: graphClient,
-			}
 			defer wg.Done()
-			runner := checker.Runner{CheckRequest: c}
+			runner := checker.Runner{CheckRequest: request}
 			resultsCh <- runner.Run(checkFn)
 		}()
 	}
 	wg.Wait()
 	close(resultsCh)
 }
 
-func RunScorecards(ctx context.Context, logger *zap.SugaredLogger,
-	repo repos.RepoURL, checksToRun checker.CheckNameToFnMap) repos.RepoResult {
+func RunScorecards(ctx context.Context,
+	repo repos.RepoURL,
+	checksToRun checker.CheckNameToFnMap,
+	httpClient *http.Client,
+	githubClient *github.Client,
+	graphClient *githubv4.Client) repos.RepoResult {
 	ret := repos.RepoResult{
 		Repo: repo.URL(),
 		Date: time.Now().Format("2006-01-02"),
 	}
 	resultsCh := make(chan checker.CheckResult)
-	go runEnabledChecks(ctx, logger, repo, checksToRun, resultsCh)
+	go runEnabledChecks(ctx, repo, checksToRun,
+		httpClient, githubClient, graphClient,
+		resultsCh)
 	for result := range resultsCh {
 		ret.Checks = append(ret.Checks, result)
 	}