From 112480952b6cb69917aa102aa14f35f360b1f75c Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Sun, 21 Jan 2024 16:42:30 -0500 Subject: [PATCH] add Makefile and run benchmarks as part of PRs (#130) This: - Adds a Makefile with `make test` and `make lint` and `make bench` - Adds a new workflow for `main` that runs benchmarks and saves them - Adds a new workflow for PRs that runs benchmarks and compares against `main`. When a benchmark regression occurs, it should fail the workflow --- .github/workflows/bench.yml | 80 +++++++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 52 ++++++++++++++++++++++++ Makefile | 24 +++++++++++ 3 files changed, 156 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 .github/workflows/main.yml create mode 100644 Makefile diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..302d303 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,80 @@ +name: Benchmark + +on: + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + go-bench: + strategy: + matrix: + go-version: [ '1.19', 'stable' ] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # to be able to retrieve the last commit in main + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Run benchmark and store the output to a file + run: | + set -o pipefail + make bench | tee ${{ github.sha }}_bench_output.txt + + - name: Get CPU information + uses: kenchan0130/actions-system-info@v1.2.1 + id: system-info + + - name: Get Main branch SHA + id: get-main-branch-sha + run: | + SHA=$(git rev-parse origin/main) + echo "sha=$SHA" >> $GITHUB_OUTPUT + + - name: Get benchmark JSON from main branch + id: cache + uses: actions/cache/restore@v3 + with: + path: ./cache/benchmark-data.json + key: ${{ steps.get-main-branch-sha.outputs.sha }}-${{ runner.os }}-${{ steps.system-info.outputs.cpu-model }}-go-benchmark + + - name: Compare benchmarks with Main + uses: benchmark-action/github-action-benchmark@v1 + if: steps.cache.outputs.cache-hit == 'true' + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: ${{ github.sha }}_bench_output.txt + # Where the benchmarks in main are (to compare) + external-data-json-path: ./cache/benchmark-data.json + # Do not save the data + save-data-file: false + # Workflow will fail when an alert happens + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable Job Summary for PRs + summary-always: true + + - name: Run benchmarks but don't compare to Main branch + uses: benchmark-action/github-action-benchmark@v1 + if: steps.cache.outputs.cache-hit != 'true' + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: ${{ github.sha }}_bench_output.txt + # Write benchmarks to this file, do not publish to GitHub Pages + save-data-file: false + external-data-json-path: ./cache/benchmark-data.json + # Enable Job Summary for PRs + summary-always: true \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..157e612 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,52 @@ +name: Main +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + go-bench: + strategy: + matrix: + go-version: [ '1.19', 'stable' ] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Run benchmark and store the output to a file + run: | + set -o pipefail + make bench | tee bench_output.txt + + - name: Get benchmark as JSON + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: bench_output.txt + # Write benchmarks to this file + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get CPU information + uses: kenchan0130/actions-system-info@v1.2.1 + id: system-info + + - name: Save benchmark JSON to cache + uses: actions/cache/save@v3 + with: + path: ./cache/benchmark-data.json + # Save with commit hash to avoid "cache already exists" + # Save with OS & CPU info to prevent comparing against results from different CPUs + key: ${{ github.sha }}-${{ runner.os }}-${{ steps.system-info.outputs.cpu-model }}-go-benchmark \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3e0720a --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +.DEFAULT_GOAL := help + +GO_BIN ?= $(shell go env GOPATH)/bin + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +$(GO_BIN)/golangci-lint: + @echo "==> Installing golangci-lint within "${GO_BIN}"" + @go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +.PHONY: lint +lint: $(GO_BIN)/golangci-lint ## Run linting on Go files + @echo "==> Linting Go source files" + @golangci-lint run -v --fix -c .golangci.yml ./... + +.PHONY: test +test: ## Run tests + go test -race -v ./... -coverprofile ./coverage.txt + +.PHONY: bench +bench: ## Run benchmarks. See https://pkg.go.dev/cmd/go#hdr-Testing_flags + go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -cpu 1 -benchmem