From 9ad5dd0fd8abdd9ce27a83fa3cf7f31375c1fa39 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 2 Aug 2023 17:36:17 +0200 Subject: [PATCH 1/7] feat: add version flag --- .github/workflows/release-docker.yml | 6 ++++++ CHANGELOG.md | 2 ++ Dockerfile | 3 ++- Makefile | 8 ++++++-- cmd/gateway-conformance/main.go | 5 +++-- tooling/env.go | 4 ++++ 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 4f7f922ec..bd4ff1375 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -2,6 +2,9 @@ name: Release Docker on: workflow_dispatch: inputs: + tag: + description: 'The tag that is being released.' + required: false tags: description: 'Comma separated list of tags to apply to the image.' required: false @@ -27,6 +30,7 @@ jobs: outputs: draft: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].draft || false }} tags: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tags || github.event.inputs.tags }} + tag: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tag || github.event.inputs.tag }} steps: # This step will download the release artifact either from the workflow # run that triggered this workflow or from the artifacts-url input. It @@ -65,6 +69,8 @@ jobs: - uses: docker/build-push-action@v4 with: context: . + build-args: | + VERSION:${{ needs.release.outputs.tag }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags.outputs.tags }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9e8af86..42e5c56bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- `--version` flag shows the current version ## [0.3.0] - 2023-07-31 ### Added diff --git a/Dockerfile b/Dockerfile index b272e978b..468f2b208 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ COPY ./go.mod ./go.sum ./ RUN go mod download COPY . . -RUN go build -o ./gateway-conformance ./cmd/gateway-conformance +ARG VERSION=dev +RUN go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=${VERSION}" -o ./gateway-conformance ./cmd/gateway-conformance ENTRYPOINT ["/app/gateway-conformance"] diff --git a/Makefile b/Makefile index a4c4743c7..61db8800e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +GIT_COMMIT := $(shell git rev-parse --short HEAD) +DIRTY_SUFFIX := $(shell test -n "`git status --porcelain`" && echo "-dirty" || true) +CLI_VERSION := dev-$(GIT_COMMIT)$(DIRTY_SUFFIX) + all: gateway-conformance clean: clean-docker @@ -30,7 +34,7 @@ fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. gateway-conformance: - go build -o ./gateway-conformance ./cmd/gateway-conformance + go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=$(CLI_VERSION)" -o ./gateway-conformance ./cmd/gateway-conformance test-docker: docker fixtures.car gateway-conformance ./gc test @@ -44,7 +48,7 @@ test-docker: docker fixtures.car gateway-conformance open ./reports/output.html docker: - docker build -t gateway-conformance . + docker build --build-arg VERSION="$(CLI_VERSION)" -t gateway-conformance . clean-docker: @if command -v docker >/dev/null 2>&1 && docker image inspect gateway-conformance >/dev/null 2>&1; then \ diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index b6f0b4f98..8aab203d5 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -78,8 +78,9 @@ func main() { var verbose bool app := &cli.App{ - Name: "gateway-conformance", - Usage: "Tooling for the gateway test suite", + Name: "gateway-conformance", + Usage: "Tooling for the gateway test suite", + Version: tooling.Version, Commands: []*cli.Command{ { Name: "test", diff --git a/tooling/env.go b/tooling/env.go index cb0765cd7..ff888f387 100644 --- a/tooling/env.go +++ b/tooling/env.go @@ -6,6 +6,10 @@ import ( "runtime" ) +var ( + Version = "dev" +) + func Home() string { home := os.Getenv("GATEWAY_CONFORMANCE_HOME") if home == "" { From 787473514d7aee2222943bcfb4b209d4ccee5645 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 2 Aug 2023 18:23:14 +0200 Subject: [PATCH 2/7] feat: add metadata --- tests/metadata_test.go | 11 +++++++++++ tooling/metadata.go | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/metadata_test.go create mode 100644 tooling/metadata.go diff --git a/tests/metadata_test.go b/tests/metadata_test.go new file mode 100644 index 000000000..09ab5ee92 --- /dev/null +++ b/tests/metadata_test.go @@ -0,0 +1,11 @@ +package tests + +import ( + "testing" + + "github.com/ipfs/gateway-conformance/tooling" +) + +func TestMetadata(t *testing.T) { + tooling.LogVersion(t) +} diff --git a/tooling/metadata.go b/tooling/metadata.go new file mode 100644 index 000000000..006fdc07d --- /dev/null +++ b/tooling/metadata.go @@ -0,0 +1,23 @@ +package tooling + +import ( + "encoding/json" + "testing" +) + +func LogMetadata(t *testing.T, value interface{}) { + jsonValue, err := json.Marshal(value) + if err != nil { + t.Errorf("Failed to encode value: %v", err) + return + } + t.Logf("--- META: %s", string(jsonValue)) +} + +func LogVersion(t *testing.T) { + LogMetadata(t, struct { + Version string `json:"version"` + }{ + Version: Version, + }) +} From a6d1c98d4c6e509ce7e7f301ed980c64ff8f004a Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Thu, 3 Aug 2023 09:02:02 +0200 Subject: [PATCH 3/7] feat: add metadata processing --- aggregate.js | 54 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/aggregate.js b/aggregate.js index 347096409..f19462e34 100644 --- a/aggregate.js +++ b/aggregate.js @@ -1,5 +1,8 @@ const fs = require("fs"); +// # we group test results by Path, depth is the number of levels to group by +const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1; + // # read json from stdin: let lines = fs.readFileSync(0, "utf-8"); lines = JSON.parse(lines); @@ -10,9 +13,44 @@ lines = lines.filter((line) => { return Test !== undefined; }); +// # extract test metadata +// action is output, and starts with ".* --- META: (.*)" +// see details in https://github.com/ipfs/gateway-conformance/pull/125 +const getMetadata = (line) => { + const { Action, Output } = line; + + if (Action !== "output") { + return null; + } + + const match = Output.match(/.* --- META: (.*)/); + + if (!match) { + return null; + } + + const metadata = match[1]; + return JSON.parse(metadata); +} + +lines = lines.map((line) => { + const metadata = getMetadata(line); + + if (!metadata) { + return line; + } + + return { + ...line, + Action: "meta", + Metadata: metadata, + } +}); + +// # keep the test result lines and metadata only lines = lines.filter((line) => { const { Action } = line; - return ["pass", "fail", "skip"].includes(Action); + return ["pass", "fail", "skip", "meta"].includes(Action); }); // # add "Path" field by parsing "Name" and split by "/" @@ -69,8 +107,6 @@ lines = lines.filter((line) => { // # Aggregate by Path and count actions -const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1; - // test result is a map { [path_str]: { [path], [action]: count } } const testResults = {}; @@ -82,12 +118,18 @@ lines.forEach((line) => { const key = path.join(" > "); if (!current[key]) { - current[key] = {Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0}; + current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} }; } current = current[key]; - current[Action] += 1; - current["total"] += 1; + if (Action === "meta") { + const { Metadata } = line; + current["meta"] = { ...current["meta"], ...Metadata }; + return; + } else { + current[Action] += 1; + current["total"] += 1; + } }); // output result to stdout From 1a269068038cbf5bcdf7a659996e68deeeb66fbb Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Thu, 3 Aug 2023 09:09:51 +0200 Subject: [PATCH 4/7] feat: output version in aggregates --- aggregate-into-table.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/aggregate-into-table.js b/aggregate-into-table.js index 67ea9012a..6994eb3dd 100644 --- a/aggregate-into-table.js +++ b/aggregate-into-table.js @@ -1,13 +1,14 @@ const fs = require("fs"); +const TestMetadata = "TestMetadata"; + // retrieve the list of input files from the command line const files = process.argv.slice(2); // read all input files (json) const inputs = files.map((file) => { return JSON.parse(fs.readFileSync(file, 'utf8')); -} -); +}); // merge all the unique keys from all the inputs let keys = new Set(); @@ -16,13 +17,14 @@ inputs.forEach((input) => { keys.add(key); }); }); +keys.delete(TestMetadata); // Extract TestMetadata which is a special case keys = Array.from(keys).sort(); // generate a table const columns = []; -// add the leading column ("gateway", "key1", "key2", ... "keyN") -const leading = ["gateway"]; +// add the leading column ("gateway", "version", "key1", "key2", ... "keyN") +const leading = ["gateway", "version"]; keys.forEach((key) => { // Skip the "Test" prefix const niceKey = key.replace(/^Test/, ''); @@ -53,7 +55,13 @@ inputs.forEach((input, index) => { // clean name (remove path and extension) let name = files[index].replace(/\.json$/, '').replace(/^.*\//, ''); - const col = [name]; + // extract TestMetadata & version + const metadata = input[TestMetadata]["meta"]; + const version = metadata['version']; + + const col = [name, version]; + + // extract results keys.forEach((key) => { col.push(cellRender(input[key] || null)); }); From 03bfc642bb83454f2a240b39a8c43c7d1e1efd8e Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Thu, 3 Aug 2023 09:56:04 +0200 Subject: [PATCH 5/7] feat: add the version to build args --- .github/actions/extract-fixtures/action.yml | 2 ++ .github/actions/test/action.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/actions/extract-fixtures/action.yml b/.github/actions/extract-fixtures/action.yml index 1d349b991..0c6ba5ded 100644 --- a/.github/actions/extract-fixtures/action.yml +++ b/.github/actions/extract-fixtures/action.yml @@ -25,3 +25,5 @@ runs: ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }} dockerfile: Dockerfile args: extract-fixtures --directory="$OUTPUT" --merged="$MERGED" + build-args: | + VERSION:${{ steps.github.outputs.action_ref }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 89664cc25..8a28e771e 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -46,6 +46,8 @@ runs: dockerfile: Dockerfile opts: --network=host args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }} + build-args: | + VERSION:${{ steps.github.outputs.action_ref }} - name: Create the XML if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success()) uses: pl-strflt/gotest-json-to-junit-xml@v1 From df4c4753cc6bdcb507fa4ea9e4ada5853d4c01a7 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 4 Aug 2023 13:07:10 +0200 Subject: [PATCH 6/7] fix: add version during testing --- cmd/gateway-conformance/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 8aab203d5..dc0bd90f7 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -127,6 +127,9 @@ func main() { args = append(args, fmt.Sprintf("-specs=%s", specs)) } + ldFlag := fmt.Sprintf("-ldflags=-X github.com/ipfs/gateway-conformance/tooling.Version=%s", tooling.Version) + args = append(args, ldFlag) + args = append(args, cCtx.Args().Slice()...) fmt.Println("go " + strings.Join(args, " ")) From d1c8ff8f94017b67e148556fb74a511d0b29fb6e Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Thu, 3 Aug 2023 09:13:48 +0200 Subject: [PATCH 7/7] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e5c56bc..fc2788da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added - `--version` flag shows the current version +- Metadata logging used to associate tests with custom data like versions, specs identifiers, etc. ## [0.3.0] - 2023-07-31 ### Added