From 8c392a4cdd0363f5150dcfabaa992b2aa7b3b741 Mon Sep 17 00:00:00 2001 From: Ilianna Papastefanou Date: Tue, 5 Apr 2022 16:00:08 +0100 Subject: [PATCH] test: get stdout failure text before exit code This commit moves all exit code assertions from the top assertion to the bottom. In this way we can see the stdout actual text first when a test fails. It will be even easier for checking it in CircleCI. --- .../acceptance/iac/cli-share-results.spec.ts | 27 ++++------ test/jest/acceptance/iac/custom-rules.spec.ts | 25 ++++----- test/jest/acceptance/iac/file-output.spec.ts | 3 +- .../acceptance/iac/iac-entitlement.spec.ts | 2 +- test/jest/acceptance/iac/report.spec.ts | 27 ---------- test/jest/acceptance/iac/test-arm.spec.ts | 12 ++--- .../iac/test-cloudformation.spec.ts | 14 +++-- .../acceptance/iac/test-directory.spec.ts | 8 +-- .../acceptance/iac/test-kubernetes.spec.ts | 16 +++--- .../iac/test-terraform-plan.spec.ts | 13 ++--- .../iac/test-terraform-var-deref.spec.ts | 51 +++++++------------ .../acceptance/iac/test-terraform.spec.ts | 14 ++--- 12 files changed, 69 insertions(+), 143 deletions(-) diff --git a/test/jest/acceptance/iac/cli-share-results.spec.ts b/test/jest/acceptance/iac/cli-share-results.spec.ts index e8841869fb..fc09884f3f 100644 --- a/test/jest/acceptance/iac/cli-share-results.spec.ts +++ b/test/jest/acceptance/iac/cli-share-results.spec.ts @@ -32,11 +32,11 @@ describe('CLI Share Results', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/arm/rule_test.json --report`, ); - expect(exitCode).toBe(2); expect(stdout).toMatch( 'Flag "--report" is only supported if feature flag "iacCliShareResults" is enabled. To enable it, please contact Snyk support.', ); + expect(exitCode).toBe(2); }); }); @@ -49,11 +49,11 @@ describe('CLI Share Results', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/arm/rule_test.json --report`, ); - expect(exitCode).toBe(1); expect(stdout).toContain( `Your test results are available at: http://localhost:${server.getPort()}/org/test-org/projects under the name snyk/cli`, ); + expect(exitCode).toBe(1); }); it('forwards value to iac-cli-share-results endpoint', async () => { @@ -61,8 +61,6 @@ describe('CLI Share Results', () => { `snyk iac test ./iac/arm/rule_test.json --report`, ); - expect(exitCode).toEqual(1); - const testRequests = server .getRequests() .filter((request) => request.url?.includes('/iac-cli-share-results')); @@ -88,6 +86,7 @@ describe('CLI Share Results', () => { ], }), ); + expect(exitCode).toEqual(1); }); it('forwards project tags', async () => { @@ -95,8 +94,6 @@ describe('CLI Share Results', () => { 'snyk iac test ./iac/arm/rule_test.json --report --tags=foo=bar', ); - expect(exitCode).toEqual(1); - const requests = server .getRequests() .filter((request) => request.url.includes('/iac-cli-share-results')); @@ -108,6 +105,7 @@ describe('CLI Share Results', () => { expect(request.body).toMatchObject({ tags: [{ key: 'foo', value: 'bar' }], }); + expect(exitCode).toEqual(1); }); it('forwards project environment', async () => { @@ -115,8 +113,6 @@ describe('CLI Share Results', () => { 'snyk iac test ./iac/arm/rule_test.json --report --project-environment=saas', ); - expect(exitCode).toEqual(1); - const requests = server .getRequests() .filter((request) => request.url.includes('/iac-cli-share-results')); @@ -130,6 +126,7 @@ describe('CLI Share Results', () => { environment: ['saas'], }, }); + expect(exitCode).toEqual(1); }); it('forwards project lifecycle', async () => { @@ -137,8 +134,6 @@ describe('CLI Share Results', () => { 'snyk iac test ./iac/arm/rule_test.json --report --project-lifecycle=sandbox', ); - expect(exitCode).toEqual(1); - const requests = server .getRequests() .filter((request) => request.url.includes('/iac-cli-share-results')); @@ -152,6 +147,7 @@ describe('CLI Share Results', () => { lifecycle: ['sandbox'], }, }); + expect(exitCode).toEqual(1); }); it('forwards project business criticality', async () => { @@ -159,21 +155,18 @@ describe('CLI Share Results', () => { 'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high', ); - expect(exitCode).toEqual(1); - const requests = server .getRequests() .filter((request) => request.url.includes('/iac-cli-share-results')); - expect(requests.length).toEqual(1); const [request] = requests; - expect(request.body).toMatchObject({ attributes: { criticality: ['high'], }, }); + expect(exitCode).toEqual(1); }); it('should print an error message if test usage is exceeded', async () => { @@ -183,8 +176,8 @@ describe('CLI Share Results', () => { 'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high', ); - expect(exitCode).toEqual(2); expect(stdout).toMatch(/test limit reached/i); + expect(exitCode).toEqual(2); }); describe('with target reference', () => { @@ -195,12 +188,9 @@ describe('CLI Share Results', () => { `snyk iac test ./iac/arm/rule_test.json --report --target-reference=${testTargetRef}`, ); - expect(exitCode).toEqual(1); - const testRequests = server .getRequests() .filter((request) => request.url?.includes('/iac-cli-share-results')); - expect(testRequests[0]).toMatchObject({ body: { contributors: expect.any(Array), @@ -221,6 +211,7 @@ describe('CLI Share Results', () => { ], }, }); + expect(exitCode).toEqual(1); }); }); }); diff --git a/test/jest/acceptance/iac/custom-rules.spec.ts b/test/jest/acceptance/iac/custom-rules.spec.ts index 2e73217199..f4dd757ac5 100644 --- a/test/jest/acceptance/iac/custom-rules.spec.ts +++ b/test/jest/acceptance/iac/custom-rules.spec.ts @@ -21,7 +21,6 @@ describe('iac test --rules', () => { const { stdout, exitCode } = await run( `snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`, ); - expect(exitCode).toBe(1); expect(stdout).toContain( 'Using custom rules to generate misconfigurations.', @@ -33,39 +32,37 @@ describe('iac test --rules', () => { expect(stdout).toContain( 'introduced by input > resource > aws_security_group[allow_ssh] > tags', ); + expect(exitCode).toBe(1); }); it('does not show custom rules message if it scans local custom rules and uses --json flag', async () => { const { stdout, exitCode } = await run( `snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf --json`, ); - expect(exitCode).toBe(1); - expect(stdout).not.toContain( 'Using custom rules to generate misconfigurations.', ); + expect(exitCode).toBe(1); }); it('presents an error message when the local rules cannot be found', async () => { const { stdout, exitCode } = await run( `snyk iac test --rules=./not/a/real/path.tar.gz ./iac/terraform/sg_open_ssh.tf`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain( 'We were unable to extract the rules provided at: ./not/a/real/path.tar.gz', ); + expect(exitCode).toBe(2); }); it('presents an error message when the user is not entitled to custom-rules', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=no-custom-rules-entitlements --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain( `Flag "--rules" is currently not supported for this org. To enable it, please contact snyk support.`, ); + expect(exitCode).toBe(2); }); describe.each([ @@ -76,9 +73,8 @@ describe('iac test --rules', () => { const { stderr, exitCode } = await run( `snyk iac ${testedCommand} --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`, ); - - expect(exitCode).toEqual(1); expect(stderr).toEqual(''); + expect(exitCode).toEqual(1); }); it('should display a message informing of the application of custom rules', async () => { @@ -219,7 +215,6 @@ describe('custom rules pull from a remote OCI registry', () => { expect(SNYK_CFG_OCI_REGISTRY_URL).toBeDefined(); expect(SNYK_CFG_OCI_REGISTRY_USERNAME).toBeDefined(); expect(SNYK_CFG_OCI_REGISTRY_PASSWORD).toBeDefined(); - expect(exitCode).toBe(1); expect(stdout).toContain( 'Using custom rules to generate misconfigurations.', @@ -231,6 +226,7 @@ describe('custom rules pull from a remote OCI registry', () => { expect(stdout).toContain( 'introduced by input > resource > aws_security_group[allow_ssh] > tags', ); + expect(exitCode).toBe(1); }); describe.each([ @@ -246,9 +242,8 @@ describe('custom rules pull from a remote OCI registry', () => { SNYK_CFG_OCI_REGISTRY_PASSWORD: SNYK_CFG_OCI_REGISTRY_PASSWORD as string, }, ); - - expect(exitCode).toEqual(1); expect(stderr).toContain(''); + expect(exitCode).toEqual(1); }); it('should display a message informing of the application of custom rules', async () => { @@ -313,10 +308,10 @@ describe('custom rules pull from a remote OCI registry', () => { }, ); - expect(exitCode).toBe(2); expect(stdout).toContain( 'There was an authentication error. Incorrect credentials provided.', ); + expect(exitCode).toBe(2); }); it('presents an error message when there is a local and remote rules conflict', async () => { @@ -328,10 +323,10 @@ describe('custom rules pull from a remote OCI registry', () => { }, ); - expect(exitCode).toBe(2); expect(stdout).toContain( 'Remote and local custom rules bundle can not be used at the same time.', ); + expect(exitCode).toBe(2); }); it('presents an error message when the user is not entitled to custom-rules', async () => { @@ -346,9 +341,9 @@ describe('custom rules pull from a remote OCI registry', () => { }, ); - expect(exitCode).toBe(2); expect(stdout).toContain( `The custom rules feature is currently not supported for this org. To enable it, please contact snyk support.`, ); + expect(exitCode).toBe(2); }); }); diff --git a/test/jest/acceptance/iac/file-output.spec.ts b/test/jest/acceptance/iac/file-output.spec.ts index cdd95dbc02..8e3e54df7d 100644 --- a/test/jest/acceptance/iac/file-output.spec.ts +++ b/test/jest/acceptance/iac/file-output.spec.ts @@ -112,8 +112,7 @@ describe('iac test --sarif-file-output', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/file-logging -d`, ); - expect(exitCode).toBe(1); - expect(stdout).not.toContain('PRIVATE_FILE_CONTENT_CHECK'); + expect(exitCode).toBe(1); }); }); diff --git a/test/jest/acceptance/iac/iac-entitlement.spec.ts b/test/jest/acceptance/iac/iac-entitlement.spec.ts index f0e3e861fd..3c1ef5e70e 100644 --- a/test/jest/acceptance/iac/iac-entitlement.spec.ts +++ b/test/jest/acceptance/iac/iac-entitlement.spec.ts @@ -18,9 +18,9 @@ describe('iac test with infrastructureAsCode entitlement', () => { const { stdout, exitCode } = await run( `snyk iac test --org=no-iac-entitlements ./iac/terraform/sg_open_ssh.tf`, ); - expect(exitCode).toBe(2); expect(stdout).toContain( 'This feature is currently not enabled for your org. To enable it, please contact snyk support.', ); + expect(exitCode).toBe(2); }); }); diff --git a/test/jest/acceptance/iac/report.spec.ts b/test/jest/acceptance/iac/report.spec.ts index 3304d72479..1fd761895e 100644 --- a/test/jest/acceptance/iac/report.spec.ts +++ b/test/jest/acceptance/iac/report.spec.ts @@ -23,32 +23,24 @@ describe('iac report', () => { afterAll(async () => teardown()); it('should return exit code 1', async () => { - // Act const { exitCode } = await run(`snyk iac report ./iac/arm/rule_test.json`); - - // Assert expect(exitCode).toEqual(1); }); it('should include test results in the output', async () => { const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`); - expect(stdout).toContain('Infrastructure as code issues:'); }); it('should include a link to the projects page in the output', async () => { const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`); - expect(stdout).toContain( `Your test results are available at: ${API_HOST}/org/test-org/projects under the name snyk/cli`, ); }); it('should forward the scan results to the /iac-cli-share-results endpoint', async () => { - // Act await run(`snyk iac report ./iac/arm/rule_test.json`); - - // Assert const testRequests = server .getRequests() .filter((request) => request.url?.includes('/iac-cli-share-results')); @@ -78,26 +70,16 @@ describe('iac report', () => { describe("when called without the 'iacCliShareResults' feature flag", () => { it('should return an error status code', async () => { - // Arrange server.setFeatureFlag('iacCliShareResults', false); - - // Act const { exitCode } = await run( `snyk iac report ./iac/arm/rule_test.json`, ); - - // Assert expect(exitCode).toBe(2); }); it('should print an appropriate error message', async () => { - // Arrange server.setFeatureFlag('iacCliShareResults', false); - - // Act const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`); - - // Assert expect(stdout).toMatch( "Feature flag 'iacCliShareResults' is not currently enabled for your org, to enable please contact snyk support", ); @@ -106,28 +88,19 @@ describe('iac report', () => { describe("when called without a preceding 'iac'", () => { it('should return an error status code', async () => { - // Act const { exitCode } = await run(`snyk report ./iac/arm/rule_test.json`); - - // Assert expect(exitCode).toBe(2); }); it('should print an appropriate error message', async () => { - // Act const { stdout } = await run(`snyk report ./iac/arm/rule_test.json`); - - // Assert expect(stdout).toContain( '"report" is not a supported command. Did you mean to use "iac report"?', ); }); it('should return an empty stderr', async () => { - // Act const { stderr } = await run(`snyk report ./iac/arm/rule_test.json`); - - // Assert expect(stderr).toEqual(''); }); }); diff --git a/test/jest/acceptance/iac/test-arm.spec.ts b/test/jest/acceptance/iac/test-arm.spec.ts index 70bd91429f..f192abe0ca 100644 --- a/test/jest/acceptance/iac/test-arm.spec.ts +++ b/test/jest/acceptance/iac/test-arm.spec.ts @@ -20,8 +20,6 @@ describe('ARM single file scan', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/arm/rule_test.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/arm/rule_test.json'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( @@ -30,18 +28,18 @@ describe('ARM single file scan', () => { expect(stdout).toContain( 'resources[1] > properties > networkRuleCollections[0] > properties > rules[0] > sourceAddresses', ); + expect(exitCode).toBe(1); }); it('filters out issues when using severity threshold', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/arm/rule_test.json --severity-threshold=high`, ); - - expect(exitCode).toBe(0); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( 'Tested ./iac/arm/rule_test.json for known issues, found 0 issues', ); + expect(exitCode).toBe(0); }); it('outputs an error for files with no valid JSON', async () => { @@ -49,8 +47,8 @@ describe('ARM single file scan', () => { `snyk iac test ./iac/arm/invalid_rule_test.json`, ); - expect(exitCode).toBe(2); expect(stdout).toContain('We were unable to parse the JSON file'); + expect(exitCode).toBe(2); }); it('outputs the expected text when running with --sarif flag', async () => { @@ -58,9 +56,9 @@ describe('ARM single file scan', () => { `snyk iac test ./iac/arm/rule_test.json --sarif`, ); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-TF-20",'); expect(stdout).toContain('"ruleId": "SNYK-CC-TF-20",'); + expect(exitCode).toBe(1); }); it('outputs the expected text when running with --json flag', async () => { @@ -69,9 +67,9 @@ describe('ARM single file scan', () => { ); expect(isValidJSONString(stdout)).toBe(true); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-TF-20",'); expect(stdout).toContain('"packageManager": "armconfig",'); expect(stdout).toContain('"projectType": "armconfig",'); + expect(exitCode).toBe(1); }); }); diff --git a/test/jest/acceptance/iac/test-cloudformation.spec.ts b/test/jest/acceptance/iac/test-cloudformation.spec.ts index 6970a79103..5301d21aa4 100644 --- a/test/jest/acceptance/iac/test-cloudformation.spec.ts +++ b/test/jest/acceptance/iac/test-cloudformation.spec.ts @@ -20,8 +20,6 @@ describe('CloudFormation single file scan', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/cloudformation/aurora-valid.yml`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/cloudformation/aurora-valid.yml'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( @@ -30,20 +28,20 @@ describe('CloudFormation single file scan', () => { expect(stdout).toContain( '[DocId: 0] > Resources[DatabaseAlarmTopic] > Properties > KmsMasterKeyId', ); + expect(exitCode).toBe(1); }); it('finds issues in CloudFormation JSON file', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/cloudformation/fargate-valid.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/cloudformation/fargate-valid.json'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain('✗ S3 restrict public bucket control is disabled'); expect(stdout).toContain( 'Resources[CodePipelineArtifactBucket] > Properties > PublicAccessBlockConfiguration > RestrictPublicBuckets', ); + expect(exitCode).toBe(1); }); it('filters out issues when using severity threshold', async () => { @@ -51,11 +49,11 @@ describe('CloudFormation single file scan', () => { `snyk iac test ./iac/cloudformation/aurora-valid.yml --severity-threshold=high`, ); - expect(exitCode).toBe(0); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( 'Tested ./iac/cloudformation/aurora-valid.yml for known issues, found 0 issues', ); + expect(exitCode).toBe(0); }); it('outputs an error for files with no valid YAML', async () => { @@ -63,10 +61,10 @@ describe('CloudFormation single file scan', () => { `snyk iac test ./iac/cloudformation/invalid-cfn.yml`, ); - expect(exitCode).toBe(2); expect(stdout).toContain( 'We were unable to parse the YAML file "./iac/cloudformation/invalid-cfn.yml". Please ensure that it contains properly structured YAML, without any template directives', ); + expect(exitCode).toBe(2); }); it('outputs the expected text when running with --sarif flag', async () => { @@ -75,9 +73,9 @@ describe('CloudFormation single file scan', () => { ); expect(isValidJSONString(stdout)).toBe(true); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-TF-55",'); expect(stdout).toContain('"ruleId": "SNYK-CC-TF-55",'); + expect(exitCode).toBe(1); }); it('outputs the expected text when running with --json flag', async () => { @@ -86,9 +84,9 @@ describe('CloudFormation single file scan', () => { ); expect(isValidJSONString(stdout)).toBe(true); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-TF-55",'); expect(stdout).toContain('"packageManager": "cloudformationconfig",'); expect(stdout).toContain('"projectType": "cloudformationconfig",'); + expect(exitCode).toBe(1); }); }); diff --git a/test/jest/acceptance/iac/test-directory.spec.ts b/test/jest/acceptance/iac/test-directory.spec.ts index f5c87387ca..a1b87bfca5 100644 --- a/test/jest/acceptance/iac/test-directory.spec.ts +++ b/test/jest/acceptance/iac/test-directory.spec.ts @@ -136,19 +136,19 @@ describe('Directory scan', () => { const { exitCode, stderr, stdout } = await run( `snyk iac test ./iac/kubernetes/`, ); - expect(exitCode).toBe(1); expect(stderr).toBe(''); expect(stdout).toContain( 'Tested 3 projects, 3 contained issues. Failed to test 1 project', ); + expect(exitCode).toBe(1); }); it('returns 1 even if some files failed to parse - using --json flag', async () => { const { exitCode, stderr, stdout } = await run( `snyk iac test ./iac/kubernetes/ --json`, ); - expect(exitCode).toBe(1); expect(stderr).toBe(''); expect(stdout).toContain('"ok": false'); + expect(exitCode).toBe(1); }); }); @@ -157,17 +157,17 @@ describe('Directory scan', () => { const { exitCode, stderr, stdout } = await run( `snyk iac test ./iac/no_vulnerabilities/ --severity-threshold=high`, ); - expect(exitCode).toBe(0); expect(stderr).toBe(''); expect(stdout).toContain('found 0 issues'); + expect(exitCode).toBe(0); }); it('returns 0 even if some files failed to parse - using --json flag', async () => { const { exitCode, stderr, stdout } = await run( `snyk iac test ./iac/no_vulnerabilities/ --severity-threshold=high --json`, ); - expect(exitCode).toBe(0); expect(stderr).toBe(''); expect(stdout).toContain('"ok": true'); + expect(exitCode).toBe(0); }); }); }); diff --git a/test/jest/acceptance/iac/test-kubernetes.spec.ts b/test/jest/acceptance/iac/test-kubernetes.spec.ts index 2dd31e0c9c..acee35dd14 100644 --- a/test/jest/acceptance/iac/test-kubernetes.spec.ts +++ b/test/jest/acceptance/iac/test-kubernetes.spec.ts @@ -20,37 +20,34 @@ describe('Kubernetes single file scan', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/kubernetes/pod-privileged.yaml`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/kubernetes/pod-privileged.yaml'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain('✗ Privileged container'); expect(stdout).toContain( '[DocId: 0] > input > spec > containers[example] > securityContext > privileged', ); + expect(exitCode).toBe(1); }); it('filters out issues when using severity threshold', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/kubernetes/pod-privileged.yaml --severity-threshold=high`, ); - - expect(exitCode).toBe(1); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( 'Tested ./iac/kubernetes/pod-privileged.yaml for known issues, found 1 issues', ); + expect(exitCode).toBe(1); }); it('outputs an error for files with no valid YAML', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/kubernetes/pod-invalid.yaml`, ); - - expect(exitCode).toBe(3); expect(stdout).toContain( 'Could not find any valid infrastructure as code files. Supported file extensions are tf, yml, yaml & json.', ); + expect(exitCode).toBe(3); }); it('outputs the expected text when running with --sarif flag', async () => { @@ -59,9 +56,9 @@ describe('Kubernetes single file scan', () => { ); expect(isValidJSONString(stdout)).toBe(true); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-K8S-42",'); expect(stdout).toContain('"ruleId": "SNYK-CC-K8S-42",'); + expect(exitCode).toBe(1); }); it('outputs the expected text when running with --json flag', async () => { @@ -70,20 +67,19 @@ describe('Kubernetes single file scan', () => { ); expect(isValidJSONString(stdout)).toBe(true); - expect(exitCode).toBe(1); expect(stdout).toContain('"id": "SNYK-CC-K8S-42",'); expect(stdout).toContain('"packageManager": "k8sconfig",'); expect(stdout).toContain('"projectType": "k8sconfig",'); + expect(exitCode).toBe(1); }); it('outputs an error for Helm files', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/kubernetes/helm-config.yaml`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain( 'We were unable to parse the YAML file "./iac/kubernetes/helm-config.yaml". Please ensure that it contains properly structured YAML, without any template directives', ); + expect(exitCode).toBe(2); }); }); diff --git a/test/jest/acceptance/iac/test-terraform-plan.spec.ts b/test/jest/acceptance/iac/test-terraform-plan.spec.ts index 7d1af6d0e1..745be0e172 100644 --- a/test/jest/acceptance/iac/test-terraform-plan.spec.ts +++ b/test/jest/acceptance/iac/test-terraform-plan.spec.ts @@ -20,8 +20,6 @@ describe('Terraform plan scanning', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform-plan/tf-plan-create.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/terraform-plan/tf-plan-create.json', ); @@ -30,6 +28,7 @@ describe('Terraform plan scanning', () => { expect(stdout).toContain( 'resource > aws_s3_bucket[terra_ci] > versioning > enabled', ); + expect(exitCode).toBe(1); }); it('finds issues in a Terraform plan file - explicit delta scan with flag', async () => { @@ -37,33 +36,31 @@ describe('Terraform plan scanning', () => { `snyk iac test ./iac/terraform-plan/tf-plan-create.json --scan=resource-changes`, ); - expect(exitCode).toBe(1); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( 'Tested ./iac/terraform-plan/tf-plan-create.json for known issues', ); + expect(exitCode).toBe(1); }); it('errors when a wrong value is passed to the --scan flag', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform-plan/tf-plan-create.json --scan=resrc-changes`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain( 'Unsupported value "resrc-changes" provided to flag "--scan".', ); + expect(exitCode).toBe(2); }); it('errors when no value is provided to the --scan flag', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform-plan/tf-plan-create.json.json --scan`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain( 'Unsupported value "true" provided to flag "--scan".', ); + expect(exitCode).toBe(2); }); it('succesfully scans a TF-Plan with the --json output flag', async () => { @@ -71,10 +68,10 @@ describe('Terraform plan scanning', () => { `snyk iac test ./iac/terraform-plan/tf-plan-create.json --json`, ); - expect(exitCode).toBe(1); expect(isValidJSONString(stdout)).toBe(true); expect(stdout).toContain('"id": "SNYK-CC-TF-124",'); expect(stdout).toContain('"packageManager": "terraformconfig",'); expect(stdout).toContain('"projectType": "terraformconfig",'); + expect(exitCode).toBe(1); }); }); diff --git a/test/jest/acceptance/iac/test-terraform-var-deref.spec.ts b/test/jest/acceptance/iac/test-terraform-var-deref.spec.ts index 4298a9e538..4e7b5dbe33 100644 --- a/test/jest/acceptance/iac/test-terraform-var-deref.spec.ts +++ b/test/jest/acceptance/iac/test-terraform-var-deref.spec.ts @@ -59,8 +59,6 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support iac/terraform/var_deref/sg_open_ssh.tf`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing iac/terraform/var_deref/sg_open_ssh.tf...', ); @@ -70,13 +68,13 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( 'Tested iac/terraform/var_deref/sg_open_ssh.tf for known issues', ); + expect(exitCode).toBe(1); }); it('returns error empty Terraform file', async () => { const { exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/terraform/empty_file.tf`, ); - expect(exitCode).toBe(3); }); }); @@ -85,8 +83,6 @@ describe('Terraform Language Support', () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/terraform-plan/tf-plan-create.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/terraform-plan/tf-plan-create.json', ); @@ -95,13 +91,12 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( 'resource > aws_s3_bucket[terra_ci] > versioning > enabled', ); + expect(exitCode).toBe(1); }); it('finds issues in CloudFormation YAML file', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/cloudformation/aurora-valid.yml`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/cloudformation/aurora-valid.yml', ); @@ -112,14 +107,13 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( '[DocId: 0] > Resources[DatabaseAlarmTopic] > Properties > KmsMasterKeyId', ); + expect(exitCode).toBe(1); }); it('finds issues in CloudFormation JSON file', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/cloudformation/fargate-valid.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/cloudformation/fargate-valid.json', ); @@ -130,13 +124,13 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( 'Resources[CodePipelineArtifactBucket] > Properties > PublicAccessBlockConfiguration > RestrictPublicBuckets', ); + expect(exitCode).toBe(1); }); + it('finds issues in ARM JSON file', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/arm/rule_test.json`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/arm/rule_test.json'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( @@ -145,13 +139,12 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( 'resources[1] > properties > networkRuleCollections[0] > properties > rules[0] > sourceAddresses', ); + expect(exitCode).toBe(1); }); it('finds issues in Kubernetes YAML file', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/kubernetes/pod-privileged.yaml`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/kubernetes/pod-privileged.yaml', ); @@ -160,14 +153,13 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( '[DocId: 0] > input > spec > containers[example] > securityContext > privileged', ); + expect(exitCode).toBe(1); }); it('finds issues in Kubernetes YAML multi file', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac/kubernetes/pod-privileged-multi.yaml`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( 'Testing ./iac/kubernetes/pod-privileged-multi.yaml', ); @@ -176,6 +168,7 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( '[DocId: 0] > input > spec > containers[example] > securityContext > privileged', ); + expect(exitCode).toBe(1); }); }); }); @@ -186,8 +179,6 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support ./iac/terraform/var_deref`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing sg_open_ssh.tf...'); expect( stdout.match(/✗ Security Group allows open ingress/g), @@ -204,17 +195,14 @@ describe('Terraform Language Support', () => { 'sg_open_ssh.tf', )} for known issues`, ); + expect(exitCode).toBe(1); }); - //TODO: add another test that checks a folder with edge cases - it('scans a mix of IaC files in nested directories', async () => { const { stdout, exitCode } = await run( `snyk iac test --org=tf-lang-support ./iac`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( `Testing ${path.join('kubernetes', 'pod-privileged.yaml')}`, ); @@ -256,6 +244,7 @@ describe('Terraform Language Support', () => { 'sg_open_ssh.tf', )} for known issues`, ); + expect(exitCode).toBe(1); }); }); @@ -265,8 +254,6 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support iac/terraform/sg_open_ssh.tf`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing iac/terraform/sg_open_ssh.tf...'); expect( stdout.match(/✗ Security Group allows open ingress/g), @@ -274,6 +261,7 @@ describe('Terraform Language Support', () => { expect(stdout).toContain( 'Tested iac/terraform/sg_open_ssh.tf for known issues', ); + expect(exitCode).toBe(1); }); it('filters out issues when using severity threshold', async () => { @@ -281,14 +269,13 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support iac/terraform/sg_open_ssh.tf --severity-threshold=high`, ); - // expect exitCode to be 0 or 1 - expect(exitCode).toBeLessThanOrEqual(1); - expect(stdout).toContain('Testing iac/terraform/sg_open_ssh.tf...'); expect(stdout.match(/✗ Security Group allows open ingress/g)).toBeNull(); expect(stdout).toContain( 'Tested iac/terraform/sg_open_ssh.tf for known issues', ); + // expect exitCode to be 0 or 1 + expect(exitCode).toBeLessThanOrEqual(1); }); it('filters out issues when using detection depth', async () => { @@ -296,8 +283,6 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support ./iac/terraform/ --detection-depth=2`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain( `Testing ${path.join('var_deref', 'sg_open_ssh.tf')}`, ); @@ -326,6 +311,7 @@ describe('Terraform Language Support', () => { 'sg_open_ssh.tf', )} for known issues`, ); + expect(exitCode).toBe(1); }); it('outputs an error for files with invalid HCL2', async () => { @@ -333,9 +319,8 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support ./iac/terraform/sg_open_ssh_invalid_hcl2.tf`, ); - expect(exitCode).toBe(2); - expect(stdout).toContain('We were unable to parse the Terraform file'); + expect(exitCode).toBe(2); }); it('outputs the expected text when running with --sarif flag', async () => { @@ -343,11 +328,10 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support ./iac/terraform/sg_open_ssh.tf --sarif`, ); - expect(exitCode).toBe(1); - expect(isValidJSONString(stdout)).toBe(true); expect(stdout).toContain('"id": "SNYK-CC-TF-1",'); expect(stdout).toContain('"ruleId": "SNYK-CC-TF-1",'); + expect(exitCode).toBe(1); }); it('outputs the expected text when running with --json flag', async () => { @@ -355,12 +339,11 @@ describe('Terraform Language Support', () => { `snyk iac test --org=tf-lang-support ./iac/terraform/sg_open_ssh.tf --json`, ); - expect(exitCode).toBe(1); - expect(isValidJSONString(stdout)).toBe(true); expect(stdout).toContain('"id": "SNYK-CC-TF-1",'); expect(stdout).toContain('"packageManager": "terraformconfig",'); expect(stdout).toContain('"projectType": "terraformconfig",'); + expect(exitCode).toBe(1); }); }); }); diff --git a/test/jest/acceptance/iac/test-terraform.spec.ts b/test/jest/acceptance/iac/test-terraform.spec.ts index a8f777b826..f11504f105 100644 --- a/test/jest/acceptance/iac/test-terraform.spec.ts +++ b/test/jest/acceptance/iac/test-terraform.spec.ts @@ -20,14 +20,13 @@ describe('Terraform single file scan', () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform/sg_open_ssh.tf`, ); - expect(exitCode).toBe(1); - expect(stdout).toContain('Testing ./iac/terraform/sg_open_ssh.tf'); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain('✗ Security Group allows open ingress'); expect(stdout).toContain( ' input > resource > aws_security_group[allow_ssh] > ingress', ); + expect(exitCode).toBe(1); }); it('filters out issues when using severity threshold', async () => { @@ -35,42 +34,39 @@ describe('Terraform single file scan', () => { `snyk iac test ./iac/terraform/sg_open_ssh.tf --severity-threshold=high`, ); - expect(exitCode).toBe(0); expect(stdout).toContain('Infrastructure as code issues:'); expect(stdout).toContain( 'Tested ./iac/terraform/sg_open_ssh.tf for known issues, found 0 issues', ); + expect(exitCode).toBe(0); }); it('outputs an error for files with invalid HCL2', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform/sg_open_ssh_invalid_hcl2.tf`, ); - - expect(exitCode).toBe(2); expect(stdout).toContain('We were unable to parse the Terraform file'); + expect(exitCode).toBe(2); }); it('outputs the expected text when running with --sarif flag', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform/sg_open_ssh.tf --sarif`, ); - - expect(exitCode).toBe(1); expect(isValidJSONString(stdout)).toBe(true); expect(stdout).toContain('"id": "SNYK-CC-TF-1",'); expect(stdout).toContain('"ruleId": "SNYK-CC-TF-1",'); + expect(exitCode).toBe(1); }); it('outputs the expected text when running with --json flag', async () => { const { stdout, exitCode } = await run( `snyk iac test ./iac/terraform/sg_open_ssh.tf --json`, ); - - expect(exitCode).toBe(1); expect(isValidJSONString(stdout)).toBe(true); expect(stdout).toContain('"id": "SNYK-CC-TF-1",'); expect(stdout).toContain('"packageManager": "terraformconfig",'); expect(stdout).toContain('"projectType": "terraformconfig",'); + expect(exitCode).toBe(1); }); });