diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2b9a98ab3..728717115 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,7 @@ jobs: earthly --allow-privileged ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }} - ./test/performance+run --args="-benchtime 10s --ledger.url=http://localhost:3068 --parallelism=5" + ./test/performance+run --args="-benchtime 10s --ledger.url=http://localhost:3068 --parallelism=5" --locally=yes - run: > earthly --allow-privileged diff --git a/Earthfile b/Earthfile index e6fee9256..0c094ac6e 100644 --- a/Earthfile +++ b/Earthfile @@ -9,6 +9,9 @@ CACHE --sharing=shared --id go-mod-cache /go/pkg/mod CACHE --sharing=shared --id golangci-cache /root/.cache/golangci-lint CACHE --sharing=shared --id go-cache /root/.cache/go-build +postgres: + FROM postgres:15-alpine + sources: FROM core+builder-image CACHE --id go-mod-cache /go/pkg/mod @@ -86,7 +89,7 @@ tests: IF [ "$includeIntegrationTests" = "true" ] SET goFlags="$goFlags -tags it" - WITH DOCKER --pull=postgres:15-alpine + WITH DOCKER --load=postgres:15-alpine=+postgres RUN go test $goFlags ./... END ELSE @@ -139,37 +142,6 @@ pre-commit: BUILD +generate-client BUILD +export-docs-events -bench: - FROM +tidy - RUN go install golang.org/x/perf/cmd/benchstat@latest - WORKDIR /src/test/performance - ARG benchTime=1s - ARG count=1 - ARG GOPROXY - ARG testTimeout=10m - ARG bench=. - ARG verbose=0 - ARG GOMAXPROCS=2 - ARG GOMEMLIMIT=1024MiB - LET additionalArgs="" - IF [ "$verbose" = "1" ] - SET additionalArgs=-v - END - WITH DOCKER --pull postgres:15-alpine - RUN go test -timeout $testTimeout -bench=$bench -run ^$ -tags it $additionalArgs \ - -benchtime=$benchTime | tee -a /results.txt - END - RUN benchstat /results.txt - SAVE ARTIFACT /results.txt - -benchstat: - FROM core+builder-image - RUN go install golang.org/x/perf/cmd/benchstat@latest - ARG compareAgainstRevision=main - COPY --pass-args github.com/formancehq/stack/components/ledger:$compareAgainstRevision+bench/results.txt /tmp/main.txt - COPY --pass-args +bench/results.txt /tmp/branch.txt - RUN --no-cache benchstat /tmp/main.txt /tmp/branch.txt - openapi: FROM node:20-alpine RUN apk update && apk add yq @@ -218,7 +190,7 @@ export-database-schema: FROM +sources RUN go install github.com/roerohan/wait-for-it@latest COPY --dir scripts scripts - WITH DOCKER --pull postgres:15-alpine --pull schemaspy/schemaspy:6.2.4 + WITH DOCKER --load=postgres:15-alpine=+postgres --pull schemaspy/schemaspy:6.2.4 RUN ./scripts/export-database-schema.sh END SAVE ARTIFACT docs/database/_system/diagrams AS LOCAL docs/database/_system/diagrams diff --git a/test/performance/Earthfile b/test/performance/Earthfile index 424948bd7..533e54428 100644 --- a/test/performance/Earthfile +++ b/test/performance/Earthfile @@ -4,8 +4,47 @@ IMPORT github.com/formancehq/earthly:tags/v0.16.2 AS core run: LOCALLY + ARG locally=no + ARG args="-bench=." + + IF [ $locally == "yes" ] + RUN rm -f ./report/benchmark-output.txt + RUN go test -run ^$ -tags it,local -report.file ./report/report.json -timeout 60m $args . | tee -a ./report/benchmark-output.txt + ELSE + FROM ../..+tidy + + ARG GOMAXPROCS=2 + ARG GOMEMLIMIT=1024MiB + + CACHE --id go-mod-cache /go/pkg/mod + CACHE --id go-cache /root/.cache/go-build + COPY *.go test/performance + COPY --dir scripts test/performance/ + + WORKDIR test/performance + RUN mkdir -p report + + WITH DOCKER --load=postgres:15-alpine=../../+postgres + RUN go test -run ^$ -tags it,local -report.file ./report/report.json -timeout 60m $args . | tee -a ./report/benchmark-output.txt + END + + SAVE ARTIFACT ./report/report.json + SAVE ARTIFACT ./report/benchmark-output.txt + END + +compare: + FROM core+builder-image + CACHE --id go-mod-cache /go/pkg/mod + CACHE --id go-cache /root/.cache/go-build + RUN go install golang.org/x/perf/cmd/benchstat@latest + WORKDIR /src/test/performance ARG args="" - RUN go test -bench="Write" -run ^$ -tags it,local -report.file ./report/report.json -timeout 60m $args + ARG rev=main + + COPY +run/benchmark-output.txt /report/benchmark-output-local.txt + COPY github.com/formancehq/ledger:${rev}+run/benchmark-output.txt /report/benchmark-output-remote.txt + + RUN benchstat /report/benchmark-output-local.txt /report/benchmark-output-remote.txt generate-graphs: FROM core+base-image