Skip to content

Commit

Permalink
run SAST scans during pipeline run
Browse files Browse the repository at this point in the history
align with g/g and merge check and test steps into a verify step
  • Loading branch information
petersutter committed Oct 25, 2024
1 parent dd445f5 commit 8a8cfee
Show file tree
Hide file tree
Showing 10 changed files with 234 additions and 199 deletions.
21 changes: 0 additions & 21 deletions .ci/check

This file was deleted.

17 changes: 14 additions & 3 deletions .ci/pipeline_definitions
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ gardenctl-v2:
component_descriptor:
ocm_repository: europe-docker.pkg.dev/gardener-project/snapshots
steps:
check:
image: 'golang:1.23.2'
test:
verify:
image: 'golang:1.23.2'
vars:
TEST_COV: '"yes"'
build:
image: 'golang:1.23.2'
output_dir: 'binary'
Expand All @@ -35,6 +35,17 @@ gardenctl-v2:
preprocess: 'finalize'
release:
release_callback: './.ci/update_latest_version'
assets:
- type: build-step-log
step_name: verify
purposes:
- lint
- sast
- gosec
comment: |
we use gosec (linter) for SAST scans
see: https://github.com/securego/gosec
enabled by https://github.com/gardener/gardenctl-v2/pull/TODO
slack:
channel_cfgs:
- channel_name: 'C01BKP30K1U' #sap-tech-gardenctl
Expand Down
21 changes: 0 additions & 21 deletions .ci/test

This file was deleted.

32 changes: 32 additions & 0 deletions .ci/verify
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors
#
# SPDX-License-Identifier: Apache-2.0

set -o errexit
set -o pipefail

# For the check step concourse will set the following environment variables:
# MAIN_REPO_DIR - path to the main repository

if [[ -z "${MAIN_REPO_DIR}" ]]; then
export MAIN_REPO_DIR="$(readlink -f "$(dirname ${0})/..")"
else
export MAIN_REPO_DIR="$(readlink -f ${MAIN_REPO_DIR})"
fi

pushd "${MAIN_REPO_DIR}" > /dev/null

export GOLANGCI_LINT_ADDITIONAL_FLAGS="--verbose --timeout 2m"
export GO_TEST_ADDITIONAL_FLAGS="-race"

if [ "${TEST_COV+yes}" = yes ] ; then
# supposed to be run in release jobs
make verify-extended
else
# run test instead of test-cov to speed-up jobs, as coverage slows down tests significantly
make check-generate verify
fi

popd > /dev/null
42 changes: 30 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
#
# SPDX-License-Identifier: Apache-2.0

REPO_ROOT := $(shell git rev-parse --show-toplevel)

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
Expand Down Expand Up @@ -37,23 +39,34 @@ help: ## Display this help.

##@ Development

.PHONY: tidy
tidy: ## Clean up go.mod and go.sum by removing unused dependencies.
go mod tidy

.PHONY: clean
clean: ## Remove generated files and clean up directories.
@hack/clean.sh ./internal/... ./pkg/...

.PHONY: test
test: fmt lint check go-test ## Run tests.
.PHONY: gen-markdown
gen-markdown: ## Generate markdown help files
go run ./internal/gen/markdown.go

.PHONY: generate
generate: gen-markdown $(MOCKGEN) fmt ## Run go generate
@hack/generate.sh ./pkg/... ./internal/...

.PHONY: fmt
fmt: ## Run go fmt against code.
go fmt ./...

.PHONY: check-generate
check-generate: ## Verify if code generation is up-to-date by running generate and checking for changes.
@hack/check-generate.sh $(REPO_ROOT)

.PHONY: lint
lint: ## Run golangci-lint against code.
@./hack/golangci-lint.sh

.PHONY: check
check: ## Check that the generated markdown is up-to-date
@./hack/check-markdown.sh

.PHONY: sast
sast: $(GOSEC)
@./hack/sast.sh
Expand All @@ -62,17 +75,22 @@ sast: $(GOSEC)
sast-report: $(GOSEC)
@./hack/sast.sh --gosec-report true

.PHONY: test
test: fmt lint check-markdown go-test sast ## Run tests.

.PHONY: check-markdown
check-markdown: ## Check that the generated markdown is up-to-date
@./hack/check-markdown.sh

.PHONY: go-test
go-test: ## Run go tests.
@./hack/test-integration.sh

.PHONY: gen-markdown
gen-markdown: ## Generate markdown help files
go run ./internal/gen/markdown.go
.PHONY: verify ## Run basic verification including linting, tests, static analysis and check if the generated markdown is up-to-date.
verify: lint go-test sast check-markdown

.PHONY: generate-sequential
generate-sequential: gen-markdown $(MOCKGEN) ## Run go generate
@hack/generate.sh ./pkg/... ./internal/...
.PHONY: verify-extended ## Run extended verification including code generation check, linting, tests, and detailed static analysis report.
verify-extended: check-generate check-markdown lint go-test sast-report

##@ Build

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.19.0
golang.org/x/crypto v0.28.0
golang.org/x/term v0.25.0
k8s.io/api v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/cli-runtime v0.31.1
Expand Down Expand Up @@ -124,7 +125,6 @@ require (
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/term v0.25.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.26.0 // indirect
Expand Down
130 changes: 130 additions & 0 deletions hack/check-generate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
#
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors
#
# SPDX-License-Identifier: Apache-2.0

set -e

echo "> Generate"

makefile="$1/Makefile"
check_branch="__check"
initialized_git=false
stashed=false
checked_out=false
generated=false

function delete-check-branch {
git rev-parse --verify "$check_branch" &>/dev/null && git branch -q -D "$check_branch" || :
}

function cleanup {
if [[ "$generated" == true ]]; then
if ! clean_err="$(make -f "$makefile" clean && git reset --hard -q && git clean -qdf)"; then
echo "Could not clean: $clean_err"
fi
fi

if [[ "$checked_out" == true ]]; then
if ! checkout_err="$(git checkout -q -)"; then
echo "Could not checkout to previous branch: $checkout_err"
fi
fi

if [[ "$stashed" == true ]]; then
if ! stash_err="$(git stash pop -q)"; then
echo "Could not pop stash: $stash_err"
fi
fi

if [[ "$initialized_git" == true ]]; then
if ! rm_err="$(rm -rf .git)"; then
echo "Could not delete git directory: $rm_err"
fi
fi

delete-check-branch
}

trap cleanup EXIT SIGINT SIGTERM

if which git &>/dev/null; then
if ! git rev-parse --git-dir &>/dev/null; then
initialized_git=true
git init -q
git add --all
git config --global user.name 'Gardener'
git config --global user.email 'gardener@cloud'
git commit -q --allow-empty -m 'initial commit'
fi

if [[ "$(git rev-parse --abbrev-ref HEAD)" == "$check_branch" ]]; then
echo "Already on check branch, aborting"
exit 1
fi
delete-check-branch

if [[ "$(git status -s)" != "" ]]; then
stashed=true
git stash --include-untracked -q
git stash apply -q &>/dev/null
fi

checked_out=true
git checkout -q -b "$check_branch"
git add --all
git commit -q --allow-empty -m 'checkpoint'

old_status="$(git status -s)"
if ! out=$(make -f "$makefile" clean 2>&1); then
echo "Error during calling make clean: $out"
exit 1
fi

echo ">> make generate"
generated=true
if ! out=$(make -f "$makefile" generate 2>&1); then
echo "Error during calling make generate: $out"
exit 1
fi
new_status="$(git status -s)"

if [[ "$old_status" != "$new_status" ]]; then
echo "make generate needs to be run:"
echo "$new_status"
exit 1
fi

repo_root="$(git rev-parse --show-toplevel)"
if [[ -d "$repo_root/vendor" ]]; then
echo ">> make revendor"
if ! out=$(make -f "$makefile" revendor 2>&1); then
echo "Error during calling make revendor: $out"
exit 1
fi
new_status="$(git status -s)"

if [[ "$old_status" != "$new_status" ]]; then
echo "make revendor needs to be run:"
echo "$new_status"
exit 1
fi
else
echo ">> make tidy"
if ! out=$(make -f "$makefile" tidy 2>&1); then
echo "Error during calling make tidy: $out"
exit 1
fi
new_status="$(git status -s)"

if [[ "$old_status" != "$new_status" ]]; then
echo "make tidy needs to be run:"
echo "$new_status"
exit 1
fi
fi
else
echo "No git detected, cannot run make check-generate"
fi
exit 0
24 changes: 24 additions & 0 deletions hack/clean.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
#
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors
#
# SPDX-License-Identifier: Apache-2.0

set -e

echo "> Clean"

for source_tree in $@; do
find "$(dirname "$source_tree")" -type f -name "zz_*.go" -exec rm '{}' \;
find "$(dirname "$source_tree")" -type f -name "generated.proto" -exec rm '{}' \;
find "$(dirname "$source_tree")" -type f -name "generated.pb.go" -exec rm '{}' \;
find "$(dirname "$source_tree")" -type f -name "openapi_generated.go" -exec rm '{}' \;
grep -lr '// Code generated by MockGen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
grep -lr '// Code generated by client-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
grep -lr '// Code generated by informer-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
grep -lr '// Code generated by lister-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
done

if [ -d "$PWD/docs/api-reference" ]; then
find ./docs/api-reference/ -type f -name "*.md" ! -name "README.md" -exec rm '{}' \;
fi
4 changes: 3 additions & 1 deletion hack/golangci-lint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ fi
# renovate: datasource=github-releases depName=golangci/golangci-lint
golangci_lint_version=v1.61.0

GOLANGCI_LINT_ADDITIONAL_FLAGS=${GOLANGCI_LINT_ADDITIONAL_FLAGS:-""}

# Install golangci-lint (linting tool)
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin "$golangci_lint_version"

cd "$SOURCE_PATH"

echo '> Run golangci-lint'

golangci-lint -v run ./...
golangci-lint -v run ./... ${GOLANGCI_LINT_ADDITIONAL_FLAGS}
Loading

0 comments on commit 8a8cfee

Please sign in to comment.