diff --git a/.github/actions/extract-fixtures/action.yml b/.github/actions/extract-fixtures/action.yml index 1d349b991..0c6ba5ded 100644 --- a/.github/actions/extract-fixtures/action.yml +++ b/.github/actions/extract-fixtures/action.yml @@ -25,3 +25,5 @@ runs: ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }} dockerfile: Dockerfile args: extract-fixtures --directory="$OUTPUT" --merged="$MERGED" + build-args: | + VERSION:${{ steps.github.outputs.action_ref }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 89664cc25..8a28e771e 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -46,6 +46,8 @@ runs: dockerfile: Dockerfile opts: --network=host args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }} + build-args: | + VERSION:${{ steps.github.outputs.action_ref }} - name: Create the XML if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success()) uses: pl-strflt/gotest-json-to-junit-xml@v1 diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 4f7f922ec..bd4ff1375 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -2,6 +2,9 @@ name: Release Docker on: workflow_dispatch: inputs: + tag: + description: 'The tag that is being released.' + required: false tags: description: 'Comma separated list of tags to apply to the image.' required: false @@ -27,6 +30,7 @@ jobs: outputs: draft: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].draft || false }} tags: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tags || github.event.inputs.tags }} + tag: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tag || github.event.inputs.tag }} steps: # This step will download the release artifact either from the workflow # run that triggered this workflow or from the artifacts-url input. It @@ -65,6 +69,8 @@ jobs: - uses: docker/build-push-action@v4 with: context: . + build-args: | + VERSION:${{ needs.release.outputs.tag }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags.outputs.tags }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9e8af86..fc2788da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- `--version` flag shows the current version +- Metadata logging used to associate tests with custom data like versions, specs identifiers, etc. ## [0.3.0] - 2023-07-31 ### Added diff --git a/Dockerfile b/Dockerfile index b272e978b..468f2b208 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ COPY ./go.mod ./go.sum ./ RUN go mod download COPY . . -RUN go build -o ./gateway-conformance ./cmd/gateway-conformance +ARG VERSION=dev +RUN go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=${VERSION}" -o ./gateway-conformance ./cmd/gateway-conformance ENTRYPOINT ["/app/gateway-conformance"] diff --git a/Makefile b/Makefile index a4c4743c7..61db8800e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +GIT_COMMIT := $(shell git rev-parse --short HEAD) +DIRTY_SUFFIX := $(shell test -n "`git status --porcelain`" && echo "-dirty" || true) +CLI_VERSION := dev-$(GIT_COMMIT)$(DIRTY_SUFFIX) + all: gateway-conformance clean: clean-docker @@ -30,7 +34,7 @@ fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. gateway-conformance: - go build -o ./gateway-conformance ./cmd/gateway-conformance + go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=$(CLI_VERSION)" -o ./gateway-conformance ./cmd/gateway-conformance test-docker: docker fixtures.car gateway-conformance ./gc test @@ -44,7 +48,7 @@ test-docker: docker fixtures.car gateway-conformance open ./reports/output.html docker: - docker build -t gateway-conformance . + docker build --build-arg VERSION="$(CLI_VERSION)" -t gateway-conformance . clean-docker: @if command -v docker >/dev/null 2>&1 && docker image inspect gateway-conformance >/dev/null 2>&1; then \ diff --git a/aggregate-into-table.js b/aggregate-into-table.js index 67ea9012a..6994eb3dd 100644 --- a/aggregate-into-table.js +++ b/aggregate-into-table.js @@ -1,13 +1,14 @@ const fs = require("fs"); +const TestMetadata = "TestMetadata"; + // retrieve the list of input files from the command line const files = process.argv.slice(2); // read all input files (json) const inputs = files.map((file) => { return JSON.parse(fs.readFileSync(file, 'utf8')); -} -); +}); // merge all the unique keys from all the inputs let keys = new Set(); @@ -16,13 +17,14 @@ inputs.forEach((input) => { keys.add(key); }); }); +keys.delete(TestMetadata); // Extract TestMetadata which is a special case keys = Array.from(keys).sort(); // generate a table const columns = []; -// add the leading column ("gateway", "key1", "key2", ... "keyN") -const leading = ["gateway"]; +// add the leading column ("gateway", "version", "key1", "key2", ... "keyN") +const leading = ["gateway", "version"]; keys.forEach((key) => { // Skip the "Test" prefix const niceKey = key.replace(/^Test/, ''); @@ -53,7 +55,13 @@ inputs.forEach((input, index) => { // clean name (remove path and extension) let name = files[index].replace(/\.json$/, '').replace(/^.*\//, ''); - const col = [name]; + // extract TestMetadata & version + const metadata = input[TestMetadata]["meta"]; + const version = metadata['version']; + + const col = [name, version]; + + // extract results keys.forEach((key) => { col.push(cellRender(input[key] || null)); }); diff --git a/aggregate.js b/aggregate.js index 347096409..f19462e34 100644 --- a/aggregate.js +++ b/aggregate.js @@ -1,5 +1,8 @@ const fs = require("fs"); +// # we group test results by Path, depth is the number of levels to group by +const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1; + // # read json from stdin: let lines = fs.readFileSync(0, "utf-8"); lines = JSON.parse(lines); @@ -10,9 +13,44 @@ lines = lines.filter((line) => { return Test !== undefined; }); +// # extract test metadata +// action is output, and starts with ".* --- META: (.*)" +// see details in https://github.com/ipfs/gateway-conformance/pull/125 +const getMetadata = (line) => { + const { Action, Output } = line; + + if (Action !== "output") { + return null; + } + + const match = Output.match(/.* --- META: (.*)/); + + if (!match) { + return null; + } + + const metadata = match[1]; + return JSON.parse(metadata); +} + +lines = lines.map((line) => { + const metadata = getMetadata(line); + + if (!metadata) { + return line; + } + + return { + ...line, + Action: "meta", + Metadata: metadata, + } +}); + +// # keep the test result lines and metadata only lines = lines.filter((line) => { const { Action } = line; - return ["pass", "fail", "skip"].includes(Action); + return ["pass", "fail", "skip", "meta"].includes(Action); }); // # add "Path" field by parsing "Name" and split by "/" @@ -69,8 +107,6 @@ lines = lines.filter((line) => { // # Aggregate by Path and count actions -const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1; - // test result is a map { [path_str]: { [path], [action]: count } } const testResults = {}; @@ -82,12 +118,18 @@ lines.forEach((line) => { const key = path.join(" > "); if (!current[key]) { - current[key] = {Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0}; + current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} }; } current = current[key]; - current[Action] += 1; - current["total"] += 1; + if (Action === "meta") { + const { Metadata } = line; + current["meta"] = { ...current["meta"], ...Metadata }; + return; + } else { + current[Action] += 1; + current["total"] += 1; + } }); // output result to stdout diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index b6f0b4f98..dc0bd90f7 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -78,8 +78,9 @@ func main() { var verbose bool app := &cli.App{ - Name: "gateway-conformance", - Usage: "Tooling for the gateway test suite", + Name: "gateway-conformance", + Usage: "Tooling for the gateway test suite", + Version: tooling.Version, Commands: []*cli.Command{ { Name: "test", @@ -126,6 +127,9 @@ func main() { args = append(args, fmt.Sprintf("-specs=%s", specs)) } + ldFlag := fmt.Sprintf("-ldflags=-X github.com/ipfs/gateway-conformance/tooling.Version=%s", tooling.Version) + args = append(args, ldFlag) + args = append(args, cCtx.Args().Slice()...) fmt.Println("go " + strings.Join(args, " ")) diff --git a/tests/metadata_test.go b/tests/metadata_test.go new file mode 100644 index 000000000..09ab5ee92 --- /dev/null +++ b/tests/metadata_test.go @@ -0,0 +1,11 @@ +package tests + +import ( + "testing" + + "github.com/ipfs/gateway-conformance/tooling" +) + +func TestMetadata(t *testing.T) { + tooling.LogVersion(t) +} diff --git a/tooling/env.go b/tooling/env.go index cb0765cd7..ff888f387 100644 --- a/tooling/env.go +++ b/tooling/env.go @@ -6,6 +6,10 @@ import ( "runtime" ) +var ( + Version = "dev" +) + func Home() string { home := os.Getenv("GATEWAY_CONFORMANCE_HOME") if home == "" { diff --git a/tooling/metadata.go b/tooling/metadata.go new file mode 100644 index 000000000..006fdc07d --- /dev/null +++ b/tooling/metadata.go @@ -0,0 +1,23 @@ +package tooling + +import ( + "encoding/json" + "testing" +) + +func LogMetadata(t *testing.T, value interface{}) { + jsonValue, err := json.Marshal(value) + if err != nil { + t.Errorf("Failed to encode value: %v", err) + return + } + t.Logf("--- META: %s", string(jsonValue)) +} + +func LogVersion(t *testing.T) { + LogMetadata(t, struct { + Version string `json:"version"` + }{ + Version: Version, + }) +}