From 3abbac3a45667877a861ce5ff06b0d384941c764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 2 May 2023 07:59:32 +0200 Subject: [PATCH 1/4] chore: run main pipeline after the golangci-lint --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6a08f52ed..392f653993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,10 @@ name: Main pipeline on: + workflow_run: + workflows: [golangci-lint / static-analysis] + types: + - completed push: paths-ignore: - 'mkdocs.yml' From cb3712d2f9222ad8316b83efa771597096b9f9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 3 May 2023 17:57:22 +0200 Subject: [PATCH 2/4] chore: extract modulegen pipeline to a separate workflow --- .github/workflows/ci-modulegen.yml | 59 ++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 4 -- 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci-modulegen.yml diff --git a/.github/workflows/ci-modulegen.yml b/.github/workflows/ci-modulegen.yml new file mode 100644 index 0000000000..64a7442d51 --- /dev/null +++ b/.github/workflows/ci-modulegen.yml @@ -0,0 +1,59 @@ +name: Module generator pipeline + +on: + workflow_run: + workflows: [golangci-lint / static-analysis] + types: + - completed + push: + paths-ignore: + - 'mkdocs.yml' + - 'docs/**' + - 'README.md' + pull_request: + paths-ignore: + - 'mkdocs.yml' + - 'docs/**' + - 'README.md' + +concurrency: + group: "${{ github.workflow }}-${{ github.head_ref || github.sha }}" + cancel-in-progress: true + +jobs: + test: + strategy: + matrix: + go-version: [1.19.x, 1.x] + platform: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.platform }} + env: + TESTCONTAINERS_RYUK_DISABLED: "false" + steps: + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: modVerify + working-directory: ./modulegen + run: go mod verify + + - name: ensure compilation + working-directory: ./modulegen + run: go build + + - name: Run module generator tests + working-directory: ./modulegen + run: make test-unit + + - name: Test Summary + uses: test-summary/action@4ee9ece4bca777a38f05c8fc578ac2007fe266f7 + with: + paths: "**/TEST-*.xml" + if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 392f653993..e16af48c1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,10 +59,6 @@ jobs: if: ${{ matrix.platform == 'ubuntu-latest' }} run: make test-unit - - name: Run module generator tests - if: ${{ matrix.platform == 'ubuntu-latest' }} - run: make -C modulegen test-unit - - name: Run checker run: | ./scripts/check_environment.sh From 67ae1e768e30c61c70411380591a4298abb8b842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 3 May 2023 18:13:28 +0200 Subject: [PATCH 3/4] chore: orchestrate GH workflows --- .github/workflows/README.md | 23 +++++++++++++++++++++++ .github/workflows/cockroachdb-example.yml | 4 ++++ .github/workflows/consul-example.yml | 4 ++++ .github/workflows/datastore-example.yml | 4 ++++ .github/workflows/firestore-example.yml | 4 ++++ .github/workflows/module-compose.yml | 4 ++++ .github/workflows/module-couchbase.yml | 4 ++++ .github/workflows/module-localstack.yml | 4 ++++ .github/workflows/module-mysql.yml | 4 ++++ .github/workflows/module-neo4j.yml | 4 ++++ .github/workflows/module-postgres.yml | 4 ++++ .github/workflows/module-pulsar.yml | 4 ++++ .github/workflows/module-redis.yml | 4 ++++ .github/workflows/module-vault.yml | 4 ++++ .github/workflows/mongodb-example.yml | 4 ++++ .github/workflows/nginx-example.yml | 4 ++++ .github/workflows/pubsub-example.yml | 4 ++++ .github/workflows/spanner-example.yml | 4 ++++ .github/workflows/toxiproxy-example.yml | 4 ++++ modulegen/_template/ci.yml.tmpl | 4 ++++ modulegen/main_test.go | 8 ++++---- 21 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..3e0eea18cb --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,23 @@ +# Github Workflows + +The execution order of the workflows is as follows: + +1. [Linting](golangci-lint.yml) +2. Main pipeline, will run in parallel: + - [Regular pipeline](ci.yml) + - [Reaper Off pipeline](ci-reaper-off.yml) +3. [Modules generator pipeline](ci-modulegen.yml) +4. Modules pipeline, will run in parallel: + - All modules' pipelines + - All examples' pipelines + +_MermaidJS diagram created with https://mermaid.live._ + +```mermaid +flowchart TD + A[golangci-lint] -->|Reaper-on| B(Main Pipeline) + A[golangci-lint] -->|Reaper-off| B(Main Pipeline) + B -->|Module Generator| C[Module generator] + C -->|Modules| D[All modules] + C -->|Examples| E[All examples] +``` diff --git a/.github/workflows/cockroachdb-example.yml b/.github/workflows/cockroachdb-example.yml index 82320fbb06..401daef5a5 100644 --- a/.github/workflows/cockroachdb-example.yml +++ b/.github/workflows/cockroachdb-example.yml @@ -1,6 +1,10 @@ name: Cockroachdb example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/consul-example.yml b/.github/workflows/consul-example.yml index 513fe1a218..5e33d65ed5 100644 --- a/.github/workflows/consul-example.yml +++ b/.github/workflows/consul-example.yml @@ -1,6 +1,10 @@ name: Consul example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/datastore-example.yml b/.github/workflows/datastore-example.yml index 9876c2a968..617f4ddd2b 100644 --- a/.github/workflows/datastore-example.yml +++ b/.github/workflows/datastore-example.yml @@ -1,6 +1,10 @@ name: Datastore example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/firestore-example.yml b/.github/workflows/firestore-example.yml index 547167220d..3ced6c0482 100644 --- a/.github/workflows/firestore-example.yml +++ b/.github/workflows/firestore-example.yml @@ -1,6 +1,10 @@ name: Firestore example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-compose.yml b/.github/workflows/module-compose.yml index 9d385a1950..3647d3166c 100644 --- a/.github/workflows/module-compose.yml +++ b/.github/workflows/module-compose.yml @@ -1,6 +1,10 @@ name: Compose module pipeline on: + workflow_run: + workflows: [Main pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-couchbase.yml b/.github/workflows/module-couchbase.yml index 5924370f9b..caba2fa15e 100644 --- a/.github/workflows/module-couchbase.yml +++ b/.github/workflows/module-couchbase.yml @@ -1,6 +1,10 @@ name: Couchbase module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-localstack.yml b/.github/workflows/module-localstack.yml index 8c7d4924d5..9cf584f2a0 100644 --- a/.github/workflows/module-localstack.yml +++ b/.github/workflows/module-localstack.yml @@ -1,6 +1,10 @@ name: LocalStack module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-mysql.yml b/.github/workflows/module-mysql.yml index 6b79298f77..02444ded88 100644 --- a/.github/workflows/module-mysql.yml +++ b/.github/workflows/module-mysql.yml @@ -1,6 +1,10 @@ name: MySQL module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-neo4j.yml b/.github/workflows/module-neo4j.yml index 07f4f55727..97fe5bc826 100644 --- a/.github/workflows/module-neo4j.yml +++ b/.github/workflows/module-neo4j.yml @@ -1,6 +1,10 @@ name: Neo4j module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-postgres.yml b/.github/workflows/module-postgres.yml index 0d26b5322e..7bdddf18f2 100644 --- a/.github/workflows/module-postgres.yml +++ b/.github/workflows/module-postgres.yml @@ -1,6 +1,10 @@ name: Postgres module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-pulsar.yml b/.github/workflows/module-pulsar.yml index c4c4f6234c..265a5773fb 100644 --- a/.github/workflows/module-pulsar.yml +++ b/.github/workflows/module-pulsar.yml @@ -1,6 +1,10 @@ name: Pulsar module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-redis.yml b/.github/workflows/module-redis.yml index 09da677dc4..dd45a3cab5 100644 --- a/.github/workflows/module-redis.yml +++ b/.github/workflows/module-redis.yml @@ -1,6 +1,10 @@ name: Redis module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-vault.yml b/.github/workflows/module-vault.yml index 4ad758e8ef..c75010cc51 100644 --- a/.github/workflows/module-vault.yml +++ b/.github/workflows/module-vault.yml @@ -1,6 +1,10 @@ name: vault module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/mongodb-example.yml b/.github/workflows/mongodb-example.yml index 9dd988c709..7990d2ab3f 100644 --- a/.github/workflows/mongodb-example.yml +++ b/.github/workflows/mongodb-example.yml @@ -1,6 +1,10 @@ name: MongoDB example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/nginx-example.yml b/.github/workflows/nginx-example.yml index 498075c371..15685fe9d0 100644 --- a/.github/workflows/nginx-example.yml +++ b/.github/workflows/nginx-example.yml @@ -1,6 +1,10 @@ name: Nginx example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/pubsub-example.yml b/.github/workflows/pubsub-example.yml index 3dc278fbcd..0d914fbd6f 100644 --- a/.github/workflows/pubsub-example.yml +++ b/.github/workflows/pubsub-example.yml @@ -1,6 +1,10 @@ name: Pubsub example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/spanner-example.yml b/.github/workflows/spanner-example.yml index 27fcabdb43..fd0ed2021b 100644 --- a/.github/workflows/spanner-example.yml +++ b/.github/workflows/spanner-example.yml @@ -1,6 +1,10 @@ name: Spanner example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/toxiproxy-example.yml b/.github/workflows/toxiproxy-example.yml index c8383f38db..4a6eabbfad 100644 --- a/.github/workflows/toxiproxy-example.yml +++ b/.github/workflows/toxiproxy-example.yml @@ -1,6 +1,10 @@ name: Toxiproxy example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/modulegen/_template/ci.yml.tmpl b/modulegen/_template/ci.yml.tmpl index dd5f82d137..7c2c8a8f89 100644 --- a/modulegen/_template/ci.yml.tmpl +++ b/modulegen/_template/ci.yml.tmpl @@ -1,6 +1,10 @@ {{ $lower := ToLower }}name: {{ Title }} {{ ExampleType }} pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/modulegen/main_test.go b/modulegen/main_test.go index d49651f660..3cd72aab85 100644 --- a/modulegen/main_test.go +++ b/modulegen/main_test.go @@ -471,12 +471,12 @@ func assertExampleGithubWorkflowContent(t *testing.T, example Example, exampleWo data := strings.Split(string(content), "\n") assert.Equal(t, "name: "+title+" "+example.Type()+" pipeline", data[0]) - assert.Equal(t, " test-"+lower+":", data[19]) - assert.Equal(t, " go-version: ${{ matrix.go-version }}", data[29]) - assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[36]) + assert.Equal(t, " test-"+lower+":", data[23]) + assert.Equal(t, " go-version: ${{ matrix.go-version }}", data[33]) assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[40]) assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[44]) - assert.Equal(t, " paths: \"**/TEST-"+lower+"*.xml\"", data[54]) + assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[48]) + assert.Equal(t, " paths: \"**/TEST-"+lower+"*.xml\"", data[58]) } // assert content go.mod From cb3eeaae4c978fb3be8a868ec232178449c51cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 3 May 2023 18:30:14 +0200 Subject: [PATCH 4/4] fix: remove carriage return --- modulegen/main_test.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/modulegen/main_test.go b/modulegen/main_test.go index 3cd72aab85..bd8f2ca14b 100644 --- a/modulegen/main_test.go +++ b/modulegen/main_test.go @@ -411,7 +411,7 @@ func assertExampleDocContent(t *testing.T, example Example, exampleDocFile strin lower := example.Lower() title := example.Title() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "# "+title) assert.Equal(t, data[2], `Not available until the next release of testcontainers-go :material-tag: main`) assert.Equal(t, data[4], "## Introduction") @@ -435,7 +435,7 @@ func assertExampleTestContent(t *testing.T, example Example, exampleTestFile str content, err := os.ReadFile(exampleTestFile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "package "+example.Lower()) assert.Equal(t, data[7], "func Test"+example.Title()+"(t *testing.T) {") assert.Equal(t, data[10], "\tcontainer, err := RunContainer(ctx)") @@ -451,7 +451,7 @@ func assertExampleContent(t *testing.T, example Example, exampleFile string) { exampleName := example.Title() entrypoint := example.Entrypoint() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "package "+lower) assert.Equal(t, data[8], "// "+containerName+" represents the "+exampleName+" container type used in the module") assert.Equal(t, data[9], "type "+containerName+" struct {") @@ -469,7 +469,7 @@ func assertExampleGithubWorkflowContent(t *testing.T, example Example, exampleWo lower := example.Lower() title := example.Title() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, "name: "+title+" "+example.Type()+" pipeline", data[0]) assert.Equal(t, " test-"+lower+":", data[23]) assert.Equal(t, " go-version: ${{ matrix.go-version }}", data[33]) @@ -484,7 +484,7 @@ func assertGoModContent(t *testing.T, example Example, goModFile string) { content, err := os.ReadFile(goModFile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, "module github.com/testcontainers/testcontainers-go/"+example.ParentDir()+"/"+example.Lower(), data[0]) assert.Equal(t, "\tgithub.com/testcontainers/testcontainers-go "+example.TCVersion, data[5]) } @@ -494,7 +494,7 @@ func assertMakefileContent(t *testing.T, example Example, makefile string) { content, err := os.ReadFile(makefile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[4], "\t$(MAKE) test-"+example.Lower()) } @@ -534,7 +534,23 @@ func assertToolsGoContent(t *testing.T, example Example, tools string) { content, err := os.ReadFile(tools) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[3], "// This package contains the tool dependencies of the "+example.Title()+" "+example.Type()+".") assert.Equal(t, data[5], "package tools") } + +// sanitiseContent removes the carriage return from a string +// This is needed because on Windows, we need to remove the carriage return while asserting the content +func sanitiseContent(s string) string { + fns := []func(string) string{ + func(s string) string { + return strings.ReplaceAll(s, "\r", "") + }, + } + + for _, fn := range fns { + s = fn(s) + } + + return s +}