From 31d5a7bdd311d800bc9bd80201e81364227074ce Mon Sep 17 00:00:00 2001 From: Tomasz Konieczny Date: Thu, 19 Dec 2024 15:13:48 +0100 Subject: [PATCH] feat: Example workflows - playwright examples simplified, expected-fail - timeout examples (#6078) * workflow example: expected-fail-timeout * playwright workflows simplified - shell instead of cmd/args * expected fail - timeout examples - activeDeadlineSeconds and Step timeout, playwright example fixed * expected-fail-k6-test-report-condition-always-no-artifacts updated * expected-fail-timeout-step updated --- .../executor-tests/crd-workflow/smoke.yaml | 59 +++++---------- .../edge-cases-expected-fails.yaml | 75 +++++++++++++++++-- .../test-workflows/playwright-workflow.yaml | 2 +- 3 files changed, 88 insertions(+), 48 deletions(-) diff --git a/test/playwright/executor-tests/crd-workflow/smoke.yaml b/test/playwright/executor-tests/crd-workflow/smoke.yaml index 43e5a5e3299..b85573a462e 100644 --- a/test/playwright/executor-tests/crd-workflow/smoke.yaml +++ b/test/playwright/executor-tests/crd-workflow/smoke.yaml @@ -12,6 +12,7 @@ spec: paths: - test/playwright/executor-tests/playwright-project container: + image: mcr.microsoft.com/playwright:v1.32.3-focal resources: requests: cpu: 2 @@ -19,21 +20,9 @@ spec: workingDir: /data/repo/test/playwright/executor-tests/playwright-project steps: - name: Install dependencies - run: - image: mcr.microsoft.com/playwright:v1.32.3-focal - command: - - npm - args: - - ci + shell: "npm ci" - name: Run tests - run: - image: mcr.microsoft.com/playwright:v1.32.3-focal - command: - - "npx" - args: - - "--yes" - - "playwright@1.32.3" - - "test" + shell: "npx playwright test" - name: Save artifacts workingDir: /data/repo/test/playwright/executor-tests/playwright-project artifacts: @@ -61,23 +50,10 @@ spec: workingDir: /data/repo/test/playwright/executor-tests/playwright-project steps: - name: Install dependencies - run: - image: mcr.microsoft.com/playwright:v1.32.3-focal - command: - - npm - args: - - ci + shell: "npm ci" - name: Run tests run: - image: mcr.microsoft.com/playwright:v1.32.3-focal - command: - - "npx" - args: - - "--yes" - - "playwright@1.32.3" - - "test" - - "--output" - - "/data/artifacts" + shell: "npx playwright test --output /data/artifacts" env: - name: PLAYWRIGHT_HTML_REPORT value: /data/artifacts/playwright-report @@ -91,7 +67,7 @@ spec: apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow metadata: - name: playwright-workflow-smoke-v1.32.3-shell + name: playwright-workflow-smoke-v1.32.3-command-args labels: core-tests: workflows spec: @@ -110,15 +86,22 @@ spec: steps: - name: Install dependencies run: - image: mcr.microsoft.com/playwright:v1.32.3-focal + image: mcr.microsoft.com/playwright:v1.32.3-focal # image on step level command: - npm args: - ci - name: Run tests - shell: "npx --yes playwright@1.32.3 test --output /data/artifacts && cd /data/artifacts && ls -lah" - container: + run: image: mcr.microsoft.com/playwright:v1.32.3-focal + command: + - "npx" + args: + - "--yes" + - "playwright@1.32.3" + - "test" + - "--output" + - "/data/artifacts" env: - name: PLAYWRIGHT_HTML_REPORT value: /data/artifacts/playwright-report @@ -129,7 +112,6 @@ spec: paths: - '**/*' --- ---- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow metadata: @@ -176,6 +158,7 @@ spec: paths: - test/playwright/executor-tests/playwright-project container: + image: mcr.microsoft.com/playwright:v1.32.3-focal resources: requests: cpu: 2 @@ -184,14 +167,12 @@ spec: steps: - name: Install dependencies run: - image: mcr.microsoft.com/playwright:v1.32.3-focal command: - npm args: - ci - name: Run tests run: - image: mcr.microsoft.com/playwright:v1.32.3-focal command: - "npx" args: @@ -249,7 +230,6 @@ spec: workingDir: /data/repo/test/playwright/executor-tests/playwright-project steps: - name: Install dependencies - # TODO: Upgrade our Playwright tests to use newer version; Blob reporter was not available in 1.32.3 shell: 'npm install --save-dev @playwright/test@1.38.0 && npm ci' - name: Run tests parallel: @@ -264,7 +244,7 @@ spec: - name: Merge reports condition: always shell: 'npx playwright merge-reports --reporter=html /data/reports' - artifacts: + artifacts: # artifacts with fetch from parallel steps paths: - "playwright-report/**" --- @@ -286,7 +266,6 @@ spec: workingDir: /data/repo/test/playwright/executor-tests/playwright-project steps: - name: Install dependencies - # TODO: Upgrade our Playwright tests to use newer version; Blob reporter was not available in 1.32.3 shell: 'npm install --save-dev @playwright/test@1.38.0 && npm ci' - name: Run tests parallel: @@ -295,7 +274,7 @@ spec: - from: /data/repo run: shell: 'mkdir /data/artifacts && npx playwright test --reporter=junit --shard {{ index + 1 }}/{{ count }} > /data/artifacts/junit-report{{ index + 1 }}.xml' - artifacts: + artifacts: # artifacts directly on parallel step workingDir: /data/artifacts/ paths: - '**/*' diff --git a/test/special-cases/test-workflows/edge-cases-expected-fails.yaml b/test/special-cases/test-workflows/edge-cases-expected-fails.yaml index f7df7454721..5fa0574fa5d 100644 --- a/test/special-cases/test-workflows/edge-cases-expected-fails.yaml +++ b/test/special-cases/test-workflows/edge-cases-expected-fails.yaml @@ -148,6 +148,72 @@ spec: --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow +metadata: + name: expected-fail-timeout-activedeadlineseconds + labels: + core-tests: expected-fail +spec: + job: + activeDeadlineSeconds: 1 + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + container: + resources: + requests: + cpu: 128m + memory: 128Mi + workingDir: /data/repo/test/k6/executor-tests + steps: + - name: Run test + run: + image: grafana/k6:0.43.1 + args: + - run + - k6-smoke-test.js + - -e + - K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value + env: + - name: K6_SYSTEM_ENV + value: K6_SYSTEM_ENV_value +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: expected-fail-timeout-step + labels: + core-tests: expected-fail +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/playwright/executor-tests/playwright-project + container: + image: mcr.microsoft.com/playwright:v1.32.3-focal + resources: + requests: + cpu: 2 + memory: 2Gi + workingDir: /data/repo/test/playwright/executor-tests/playwright-project + steps: + - name: Install dependencies + shell: "npm ci" + - name: Run tests + timeout: 1s + shell: "npx playwright test" + - name: Save artifacts + workingDir: /data/repo/test/playwright/executor-tests/playwright-project + artifacts: + paths: + - playwright-report/**/* +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow metadata: name: expected-fail-cypress-artifacts-condition-always-with-artifacts labels: @@ -212,14 +278,9 @@ spec: - run: args: - run - - k6-smoke-test.js - env: - - name: K6_WEB_DASHBOARD - value: "true" - - name: K6_WEB_DASHBOARD_EXPORT - value: "/data/artifacts/k6-test-report.html" + - some-incorrect-file-name.js steps: - - name: Saving artifacts + - name: Saving artifacts # should be run despite step failure condition: always workingDir: /data/artifacts artifacts: diff --git a/test/suites/test-workflows/playwright-workflow.yaml b/test/suites/test-workflows/playwright-workflow.yaml index e36fc63d4ba..c5dda4a86fb 100644 --- a/test/suites/test-workflows/playwright-workflow.yaml +++ b/test/suites/test-workflows/playwright-workflow.yaml @@ -11,7 +11,7 @@ spec: workflows: - name: playwright-workflow-smoke-v1.32.3 - name: playwright-workflow-smoke-v1.32.3-custom-report-dir - - name: playwright-workflow-smoke-v1.32.3-shell + - name: playwright-workflow-smoke-v1.32.3-command-args - name: playwright-workflow-smoke-junit - name: playwright-workflow-smoke-artifacts-double-asterisk - name: playwright-workflow-smoke-template