diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a5268d56c1..acb9bbf235 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -34,6 +34,7 @@ test/fixtures/container-app-vulns/ @snyk/mycelium test/fixtures/container-projects/ @snyk/mycelium @snyk/potion test/fixtures/docker/ @snyk/mycelium @snyk/potion test/fixtures/iac/ @snyk/group-infrastructure-as-code +test/smoke/iac/ @snyk/group-infrastructure-as-code test/smoke/spec/iac/ @snyk/group-infrastructure-as-code test/smoke/spec/snyk_code_spec.sh @snyk/zenith test/smoke/spec/snyk_basic_spec.sh @snyk/hammer @@ -70,6 +71,8 @@ test/fixtures/unmanaged-log4j-fixture @snyk/tundra test/jest/acceptance/snyk-log4shell/log4shell-detection.spec.ts @snyk/tundra test/jest/acceptance/snyk-test/app-vuln-container-project.spec.ts @snyk/mycelium /.github @snyk/hammer +/.github/workflows/iac-smoke-tests.yml @snyk/group-infrastructure-as-code +/.github/workflows/iac-smoke-tests-pulls.yml @snyk/group-infrastructure-as-code # tap tests ownership test/tap/cli-monitor/ @snyk/snyk-open-source diff --git a/.github/workflows/iac-smoke-tests-pulls.yml b/.github/workflows/iac-smoke-tests-pulls.yml new file mode 100644 index 0000000000..44db795c0c --- /dev/null +++ b/.github/workflows/iac-smoke-tests-pulls.yml @@ -0,0 +1,92 @@ +name: Infrastructure as Code Smoke Tests (Pull Requests) + +on: + pull_request: + branches: [master] + +jobs: + check_for_changed_iac_files: + name: Check for changed IaC files + runs-on: ubuntu-latest + outputs: + is_changed: ${{ steps.check_iac_files_changed.outputs.is_changed }} + + steps: + - uses: actions/checkout@v2 + + - name: Install jq + run: | + sudo apt-get install jq + + - name: Parse CODEOWNERS file + id: codeowners + uses: SvanBoxel/codeowners-action@v2.2 + with: + file_match_info: 'true' + path: ./.github/CODEOWNERS + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v29.0.4 + + - name: Get all IaC files + id: get_all_iac_files + run: | + ALL_IAC_FILES=$( + echo ${{ toJSON(steps.codeowners.outputs.filematches) }} | + jq '[ + to_entries[] | + select( + .value.owners | + index("@snyk/group-infrastructure-as-code") + ) | + .key + ]' + ) + + echo "::set-output name=all_iac_files::$( + echo $ALL_IAC_FILES + )" + + - id: check_iac_files_changed + name: Check for changed files owned by IaC + run: | + ALL_IAC_FILES=${{ toJson(steps.get_all_iac_files.outputs.all_iac_files) }} + + CHANGED_FILES=$( + echo ${{ steps.changed-files.outputs.all_changed_files }} | + jq -R 'split(" ")' + ) + + CHANGED_IAC_FILES=$( + echo $CHANGED_FILES | + jq --argjson ALL_IAC_FILES "$ALL_IAC_FILES" '[ + .[] | + . as $changed_file | + select( + $ALL_IAC_FILES | + index($changed_file) + ) + ]' + ) + + CHANGED_IAC_FILES_COUNT=$( + echo $CHANGED_IAC_FILES | jq 'length' + ) + + IS_CHANGED=$( + echo $CHANGED_IAC_FILES_COUNT | jq '. > 0' + ) + + $IS_CHANGED && + echo "Found $CHANGED_IAC_FILES_COUNT changed IaC files: $CHANGED_IAC_FILES"|| + echo "No changed IaC files found!" + + echo "::set-output name=is_changed::$IS_CHANGED" + + run_iac_smoke_tests: + name: Run IaC smoke tests + uses: ./.github/workflows/iac-smoke-tests.yml + needs: check_for_changed_iac_files + if: ${{ needs.check_for_changed_iac_files.outputs.is_changed == 'true' }} + secrets: inherit diff --git a/.github/workflows/iac-smoke-tests.yml b/.github/workflows/iac-smoke-tests.yml new file mode 100644 index 0000000000..d51fbd0a8f --- /dev/null +++ b/.github/workflows/iac-smoke-tests.yml @@ -0,0 +1,67 @@ +name: Infrastructure as Code Smoke Tests + +on: + schedule: + - cron: '0 * * * *' + release: + types: [published] + workflow_call: + +jobs: + run_iac_e2e_tests: + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos, windows] + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - uses: actions/setup-node@v3 + with: + node-version: 15 + + - name: Install jq on macOS + if: ${{ matrix.os == 'macos' }} + run: | + brew install jq + + - name: Install jq on Windows + if: ${{ matrix.os == 'windows'}} + run: | + iwr -useb get.scoop.sh -outfile 'install-scoop.ps1' + .\install-scoop.ps1 -RunAsAdmin + scoop install jq + + - name: Install jq on Ubuntu + if: ${{ matrix.os == 'ubuntu' }} + run: | + sudo apt-get install jq + + - name: Install dependencies + run: | + npm install + + - name: Build Snyk CLI + run: | + npm run build + + - name: Run IaC smoke tests - non-Windows + if: ${{ matrix.os != 'windows' }} + env: + IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }} + TEST_SNYK_COMMAND: ${{ format('node {0}/dist/cli/index.js', github.workspace) }} + run: | + npm run test:smoke:iac + + - name: Run IaC smoke tests - Windows + if: ${{ matrix.os == 'windows' }} + shell: pwsh + env: + IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }} + TEST_SNYK_COMMAND: ${{ format('node {0}\dist\cli\index.js', github.workspace) }} + run: | + npm run test:smoke:iac diff --git a/package.json b/package.json index a64bd28f7f..7460233883 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "test:unit": "jest --runInBand --testPathPattern '/test(/jest)?/unit/'", "test:acceptance": "jest --runInBand --testPathPattern '/test(/jest)?/acceptance/'", "test:tap": "tap -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register test/tap/*.test.* ", - "test:smoke": "./scripts/run-smoke-tests-locally.sh" + "test:smoke": "./scripts/run-smoke-tests-locally.sh", + "test:smoke:iac": "jest --runInBand --testPathPattern '/test/smoke(/jest)?/iac/'" }, "keywords": [ "security", diff --git a/test/smoke/iac/README.md b/test/smoke/iac/README.md new file mode 100644 index 0000000000..77c08a6113 --- /dev/null +++ b/test/smoke/iac/README.md @@ -0,0 +1,39 @@ +# Snyk Infrastructure as Code Smoke Tests + +Design goal is to have a single test suite, aligned with the scope of the Snyk CLI's smoke tests, that can detect if IaC commands do not work properly - before and after it's released. The tests help us incorporate resources and requests made via network calls, to provide better coverage for the end to end flow of these commands. Some examples: + +- Network calls made for fetching org properties, such as feature flags data, custom severities, etc. +- Downloading resources from CDNs, e.g., binary executables, ruleset bundles, etc. + +The tests were written with Jest, and use a Snyk CLI executable either configured in the PATH environment variable, or overrode, using the `TEST_SNYK_COMMAND` environment variable, see more in the 'Notes on the + +# Implementation details and usage + +These smoke tests are written with Jest, using the Snyk CLI executable identified on the runtime environment (See 'Notes on the local run' section below to read on how to override it) + +Spec in this folder is used as a + +1. **"name: Infrastructure as Code Smoke Tests" Github Action** - these run every hour and upon releases. +2. **["Infrastructure as Code Smoke Tests (Pull Requests)"] GitHub Action** - these run for pull requests to the `master` branch which include changes to files owned by group IaC. + +```sh +npm run test:smoke:iac +``` + +### Notes on the local run + +These tests can be executed with the following npm script: + +``` +npm run test:smoke:iac +``` + +Alternatively, they can be executed directly via `jest`, by running: + +``` +npx jest test/smoke/iac/ +``` + +You may specify any executable that will be used by the smoke tests, by configuring the `TEST_SNYK_COMMAND` environment variable. E.g. a local exuctable `TEST_SNYK_COMMAND="./snyk-macos"` or an `TEST_SNYK_COMMAND="npx snyk@1.500.0"` or `TEST_SNYK_COMMAND="node ./dist/cli"` for local execution. + +You may also configure an authentication token with the `SNYK_TOKEN` environment variable, to run the tests with any org and user needed. diff --git a/test/smoke/iac/test.spec.ts b/test/smoke/iac/test.spec.ts new file mode 100644 index 0000000000..0e6253c2f6 --- /dev/null +++ b/test/smoke/iac/test.spec.ts @@ -0,0 +1,28 @@ +import { run } from '../../jest/acceptance/iac/helpers'; + +jest.setTimeout(1_000 * 90); + +describe('snyk iac test --experimental', () => { + beforeAll(async () => { + await login(); + }); + + it('runs successfully and resolves with a non-error exit code', async () => { + // Arrange + const filePath = 'iac/depth_detection/root.tf'; + + // Act + const { stderr, stdout, exitCode } = await run( + `snyk iac test --experimental ${filePath}`, + ); + + // Assert + expect(stdout).toContain('Infrastructure as Code'); + expect(stderr).toBe(''); + expect(exitCode).toBeLessThan(2); + }); + + async function login() { + await run(`snyk auth ${process.env.IAC_SMOKE_TESTS_SNYK_TOKEN}`); + } +});