diff --git a/package.json b/package.json index 44a3e57529..67f154e97f 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "configstore": "^5.0.1", "debug": "^4.1.1", "diff": "^4.0.1", + "hcl-to-json": "^0.1.1", "lodash.assign": "^4.2.0", "lodash.camelcase": "^4.3.0", "lodash.clonedeep": "^4.5.0", diff --git a/src/cli/commands/test/iac-local-execution/index.ts b/src/cli/commands/test/iac-local-execution/index.ts index d009b848f5..6211f57295 100644 --- a/src/cli/commands/test/iac-local-execution/index.ts +++ b/src/cli/commands/test/iac-local-execution/index.ts @@ -1,21 +1,27 @@ import * as fs from 'fs'; -import * as YAML from 'js-yaml'; import { isLocalFolder } from '../../../../lib/detect'; import { getFileType } from '../../../../lib/iac/iac-parser'; import * as util from 'util'; import { IacFileTypes } from '../../../../lib/iac/constants'; import { IacFileScanResult, IacFileMetadata, IacFileData } from './types'; -import { buildPolicyEngine } from './policy-engine'; +import { getPolicyEngine } from './policy-engine'; import { formatResults } from './results-formatter'; +import { tryParseIacFile } from './parsers'; +import { isLocalCacheExists, REQUIRED_LOCAL_CACHE_FILES } from './local-cache'; const readFileContentsAsync = util.promisify(fs.readFile); -const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata']; // this method executes the local processing engine and then formats the results to adapt with the CLI output. // the current version is dependent on files to be present locally which are not part of the source code. // without these files this method would fail. // if you're interested in trying out the experimental local execution model for IaC scanning, please reach-out. export async function test(pathToScan: string, options) { + if (!isLocalCacheExists()) + throw Error( + `Missing IaC local cache data, please validate you have: \n${REQUIRED_LOCAL_CACHE_FILES.join( + '\n', + )}`, + ); // TODO: add support for proper typing of old TestResult interface. const results = await localProcessing(pathToScan); const formattedResults = formatResults(results, options); @@ -27,12 +33,9 @@ export async function test(pathToScan: string, options) { async function localProcessing( pathToScan: string, ): Promise { - const policyEngine = await buildPolicyEngine(); const filePathsToScan = await getFilePathsToScan(pathToScan); - const fileDataToScan = await parseFileContentsForPolicyEngine( - filePathsToScan, - ); - const scanResults = await policyEngine.scanFiles(fileDataToScan); + const fileDataToScan = await parseFilesForScan(filePathsToScan); + const scanResults = await scanFilesForIssues(fileDataToScan); return scanResults; } @@ -42,18 +45,13 @@ async function getFilePathsToScan(pathToScan): Promise { 'IaC Experimental version does not support directory scan yet.', ); } - if (getFileType(pathToScan) === 'tf') { - throw new Error( - 'IaC Experimental version does not support Terraform scan yet.', - ); - } return [ { filePath: pathToScan, fileType: getFileType(pathToScan) as IacFileTypes }, ]; } -async function parseFileContentsForPolicyEngine( +async function parseFilesForScan( filesMetadata: IacFileMetadata[], ): Promise { const parsedFileData: Array = []; @@ -62,25 +60,23 @@ async function parseFileContentsForPolicyEngine( fileMetadata.filePath, 'utf-8', ); - const yamlDocuments = YAML.safeLoadAll(fileContent); - - yamlDocuments.forEach((parsedYamlDocument, docId) => { - if ( - REQUIRED_K8S_FIELDS.every((requiredField) => - parsedYamlDocument.hasOwnProperty(requiredField), - ) - ) { - parsedFileData.push({ - ...fileMetadata, - fileContent: fileContent, - jsonContent: parsedYamlDocument, - docId, - }); - } else { - throw new Error('Invalid K8s File!'); - } - }); + const parsedFiles = tryParseIacFile(fileMetadata, fileContent); + parsedFileData.push(...parsedFiles); } return parsedFileData; } + +async function scanFilesForIssues( + parsedFiles: Array, +): Promise { + // TODO: when adding dir support move implementation to queue. + // TODO: when adding dir support gracefully handle failed scans + return Promise.all( + parsedFiles.map(async (file) => { + const policyEngine = await getPolicyEngine(file.engineType); + const scanResults = policyEngine.scanFile(file); + return scanResults; + }), + ); +} diff --git a/src/cli/commands/test/iac-local-execution/local-cache.ts b/src/cli/commands/test/iac-local-execution/local-cache.ts new file mode 100644 index 0000000000..b22cfa41ff --- /dev/null +++ b/src/cli/commands/test/iac-local-execution/local-cache.ts @@ -0,0 +1,48 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import { EngineType } from './types'; + +export const LOCAL_POLICY_ENGINE_DIR = `.iac-data`; + +const KUBERNETES_POLICY_ENGINE_WASM_PATH = path.join( + LOCAL_POLICY_ENGINE_DIR, + 'k8s_policy.wasm', +); +const KUBERNETES_POLICY_ENGINE_DATA_PATH = path.join( + LOCAL_POLICY_ENGINE_DIR, + 'k8s_data.json', +); +const TERRAFORM_POLICY_ENGINE_WASM_PATH = path.join( + LOCAL_POLICY_ENGINE_DIR, + 'tf_policy.wasm', +); +const TERRAFORM_POLICY_ENGINE_DATA_PATH = path.join( + LOCAL_POLICY_ENGINE_DIR, + 'tf_data.json', +); + +export const REQUIRED_LOCAL_CACHE_FILES = [ + KUBERNETES_POLICY_ENGINE_WASM_PATH, + KUBERNETES_POLICY_ENGINE_DATA_PATH, + TERRAFORM_POLICY_ENGINE_WASM_PATH, + TERRAFORM_POLICY_ENGINE_DATA_PATH, +]; + +export function isLocalCacheExists(): boolean { + return REQUIRED_LOCAL_CACHE_FILES.every(fs.existsSync); +} + +export function getLocalCachePath(engineType: EngineType) { + switch (engineType) { + case EngineType.Kubernetes: + return [ + `${process.cwd()}/${KUBERNETES_POLICY_ENGINE_WASM_PATH}`, + `${process.cwd()}/${KUBERNETES_POLICY_ENGINE_DATA_PATH}`, + ]; + case EngineType.Terraform: + return [ + `${process.cwd()}/${TERRAFORM_POLICY_ENGINE_WASM_PATH}`, + `${process.cwd()}/${TERRAFORM_POLICY_ENGINE_DATA_PATH}`, + ]; + } +} diff --git a/src/cli/commands/test/iac-local-execution/parsers.ts b/src/cli/commands/test/iac-local-execution/parsers.ts new file mode 100644 index 0000000000..de9e9d1da8 --- /dev/null +++ b/src/cli/commands/test/iac-local-execution/parsers.ts @@ -0,0 +1,65 @@ +import * as hclToJson from 'hcl-to-json'; +import * as YAML from 'js-yaml'; +import { EngineType, IacFileData, IacFileMetadata } from './types'; + +const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata']; + +export function tryParseIacFile( + fileMetadata: IacFileMetadata, + fileContent: string, +): Array { + switch (fileMetadata.fileType) { + case 'yaml': + case 'yml': + case 'json': + return tryParsingKubernetesFile(fileContent, fileMetadata); + case 'tf': + return [tryParsingTerraformFile(fileContent, fileMetadata)]; + default: + throw new Error('Invalid IaC file'); + } +} + +function tryParsingKubernetesFile( + fileContent: string, + fileMetadata: IacFileMetadata, +): IacFileData[] { + const yamlDocuments = YAML.safeLoadAll(fileContent); + + return yamlDocuments.map((parsedYamlDocument, docId) => { + if ( + REQUIRED_K8S_FIELDS.every((requiredField) => + parsedYamlDocument.hasOwnProperty(requiredField), + ) + ) { + return { + ...fileMetadata, + fileContent: fileContent, + jsonContent: parsedYamlDocument, + engineType: EngineType.Kubernetes, + docId, + }; + } else { + throw new Error('Invalid K8s File!'); + } + }); +} + +function tryParsingTerraformFile( + fileContent: string, + fileMetadata: IacFileMetadata, +): IacFileData { + try { + // TODO: This parser does not fail on inavlid Terraform files! it is here temporarily. + // cloud-config team will replace it to a valid parser for the beta release. + const parsedData = hclToJson(fileContent); + return { + ...fileMetadata, + fileContent: fileContent, + jsonContent: parsedData, + engineType: EngineType.Terraform, + }; + } catch (err) { + throw new Error('Invalid Terraform File!'); + } +} diff --git a/src/cli/commands/test/iac-local-execution/policy-engine.ts b/src/cli/commands/test/iac-local-execution/policy-engine.ts index 468be9e2e5..9a7fcd2e92 100644 --- a/src/cli/commands/test/iac-local-execution/policy-engine.ts +++ b/src/cli/commands/test/iac-local-execution/policy-engine.ts @@ -3,18 +3,36 @@ import { IacFileData, IacFileScanResult, PolicyMetadata, + EngineType, } from './types'; import { loadPolicy } from '@open-policy-agent/opa-wasm'; import * as fs from 'fs'; -import * as path from 'path'; +import { getLocalCachePath, LOCAL_POLICY_ENGINE_DIR } from './local-cache'; -const LOCAL_POLICY_ENGINE_DIR = `.iac-data`; -const LOCAL_POLICY_ENGINE_WASM_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}policy.wasm`; -const LOCAL_POLICY_ENGINE_DATA_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}data.json`; +export async function getPolicyEngine( + engineType: EngineType, +): Promise { + if (policyEngineCache[engineType]) { + return policyEngineCache[engineType]!; + } + + policyEngineCache[engineType] = await buildPolicyEngine(engineType); + return policyEngineCache[engineType]!; +} + +const policyEngineCache: { [key in EngineType]: PolicyEngine | null } = { + [EngineType.Kubernetes]: null, + [EngineType.Terraform]: null, +}; + +async function buildPolicyEngine( + engineType: EngineType, +): Promise { + const [ + policyEngineCoreDataPath, + policyEngineMetaDataPath, + ] = getLocalCachePath(engineType); -export async function buildPolicyEngine(): Promise { - const policyEngineCoreDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_WASM_PATH}`; - const policyEngineMetaDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_DATA_PATH}`; try { const wasmFile = fs.readFileSync(policyEngineCoreDataPath); const policyMetaData = fs.readFileSync(policyEngineMetaDataPath); @@ -44,17 +62,13 @@ class PolicyEngine { return this.opaWasmInstance.evaluate(data)[0].result; } - public async scanFiles( - filesToScan: IacFileData[], - ): Promise { + public scanFile(iacFile: IacFileData): IacFileScanResult { try { - return filesToScan.map((iacFile: IacFileData) => { - const violatedPolicies = this.evaluate(iacFile.jsonContent); - return { - ...iacFile, - violatedPolicies, - }; - }); + const violatedPolicies = this.evaluate(iacFile.jsonContent); + return { + ...iacFile, + violatedPolicies, + }; } catch (err) { // TODO: to distinguish between different failure reasons throw new Error(`Failed to run policy engine: ${err}`); diff --git a/src/cli/commands/test/iac-local-execution/results-formatter.ts b/src/cli/commands/test/iac-local-execution/results-formatter.ts index 4bbd17f75d..5a845f51f0 100644 --- a/src/cli/commands/test/iac-local-execution/results-formatter.ts +++ b/src/cli/commands/test/iac-local-execution/results-formatter.ts @@ -1,5 +1,6 @@ -import { IacFileScanResult, PolicyMetadata } from './types'; +import { EngineType, IacFileScanResult, PolicyMetadata } from './types'; import { SEVERITY } from '../../../../lib/snyk-test/common'; +import { IacProjectType } from '../../../../lib/iac/constants'; // import { // issuesToLineNumbers, // CloudConfigFileTypes, @@ -34,6 +35,11 @@ export function formatResults( // } // } +const engineTypeToProjectType = { + [EngineType.Kubernetes]: IacProjectType.K8S, + [EngineType.Terraform]: IacProjectType.TERRAFORM, +}; + function iacLocalFileScanToFormattedResult( iacFileScanResult: IacFileScanResult, severityThreshold?: SEVERITY, @@ -41,9 +47,10 @@ function iacLocalFileScanToFormattedResult( const formattedIssues = iacFileScanResult.violatedPolicies.map((policy) => { // TODO: make sure we handle this issue with annotations: // https://github.com/snyk/registry/pull/17277 - const cloudConfigPath = [`[DocId:${iacFileScanResult.docId}]`].concat( - policy.msg.split('.'), - ); + const cloudConfigPath = + iacFileScanResult.docId !== undefined + ? [`[DocId:${iacFileScanResult.docId}]`].concat(policy.msg.split('.')) + : policy.msg.split('.'); const lineNumber = -1; // TODO: once package becomes public, restore the commented out code for having the issue-to-line-number functionality // try { @@ -80,7 +87,7 @@ function iacLocalFileScanToFormattedResult( ), }, isPrivate: true, - packageManager: 'k8sconfig', + packageManager: engineTypeToProjectType[iacFileScanResult.engineType], targetFile: iacFileScanResult.filePath, }; } diff --git a/src/cli/commands/test/iac-local-execution/types.ts b/src/cli/commands/test/iac-local-execution/types.ts index 4b46229335..d0aaec34e7 100644 --- a/src/cli/commands/test/iac-local-execution/types.ts +++ b/src/cli/commands/test/iac-local-execution/types.ts @@ -5,6 +5,7 @@ export type IacFileMetadata = IacFileInDirectory; export interface IacFileData extends IacFileMetadata { jsonContent: Record; fileContent: string; + engineType: EngineType; docId?: number; } export interface IacFileScanResult extends IacFileData { @@ -16,6 +17,10 @@ export interface OpaWasmInstance { setData: (data: Record) => void; } +export enum EngineType { + Kubernetes, + Terraform, +} export interface PolicyMetadata { id: string; publicId: string; diff --git a/src/cli/commands/test/iac-output.ts b/src/cli/commands/test/iac-output.ts index bba45c02ba..100750f42e 100644 --- a/src/cli/commands/test/iac-output.ts +++ b/src/cli/commands/test/iac-output.ts @@ -30,8 +30,6 @@ function formatIacIssue( introducedBy = `\n introduced by ${pathStr}`; } - const description = extractOverview(issue.description).trim(); - const descriptionLine = `\n ${description}\n`; const severityColor = getSeveritiesColour(issue.severity); return ( @@ -43,20 +41,10 @@ function formatIacIssue( ` [${issue.id}]` + name + introducedBy + - descriptionLine + '\n' ); } -function extractOverview(description: string): string { - if (!description) { - return ''; - } - - const overviewRegExp = /## Overview([\s\S]*?)(?=##|(# Details))/m; - const overviewMatches = overviewRegExp.exec(description); - return (overviewMatches && overviewMatches[1]) || ''; -} - export function getIacDisplayedOutput( iacTest: IacTestResponse, testedInfoText: string, diff --git a/test/acceptance/cli-test/iac/cli-test.iac-k8s.spec.ts b/test/acceptance/cli-test/iac/cli-test.iac-k8s.spec.ts index 51db1cf86c..46e1eda01c 100644 --- a/test/acceptance/cli-test/iac/cli-test.iac-k8s.spec.ts +++ b/test/acceptance/cli-test/iac/cli-test.iac-k8s.spec.ts @@ -94,15 +94,13 @@ export const IacK8sTests: AcceptanceTests = { issues[2].trim().startsWith('introduced by'), 'Introduced by line', ); - t.ok(issues[3], 'description'); - t.ok(issues[4] === '', 'Empty line after description'); - t.ok(issues[5].includes('[SNYK-CC-K8S-'), 'Snyk id'); + t.ok(issues[3] === '', 'description'); + t.ok(issues[4].includes('[SNYK-CC-K8S-'), 'Snyk id'); t.ok( - issues[6].trim().startsWith('introduced by'), + issues[5].trim().startsWith('introduced by'), 'Introduced by line', ); - t.ok(issues[7], 'description'); - t.ok(issues[8] === '', 'Empty line after description'); + t.ok(issues[6] === '', 'Empty line after description'); iacTestMetaAssertions(t, res, IacAcceptanceTestType.SINGLE_K8S_FILE); } }, diff --git a/test/smoke/.iac-data/data.json b/test/smoke/.iac-data/k8s_data.json similarity index 100% rename from test/smoke/.iac-data/data.json rename to test/smoke/.iac-data/k8s_data.json diff --git a/test/smoke/.iac-data/policy.wasm b/test/smoke/.iac-data/k8s_policy.wasm similarity index 100% rename from test/smoke/.iac-data/policy.wasm rename to test/smoke/.iac-data/k8s_policy.wasm diff --git a/test/smoke/.iac-data/tf_data.json b/test/smoke/.iac-data/tf_data.json new file mode 100644 index 0000000000..0d40683704 --- /dev/null +++ b/test/smoke/.iac-data/tf_data.json @@ -0,0 +1 @@ +{".circleci":{"jobs":{"run_cspell":{"docker":[{"image":"circleci/node:stretch"}],"steps":["checkout",{"run":{"command":"make spellcheck","name":"run cspell"}}]},"run_kubeconform":{"docker":[{"image":"circleci/golang:1.14"}],"environment":{"KUBECONFORM_VERSION":"\u003c\u003c pipeline.parameters.kubeconform-version \u003e\u003e"},"steps":["checkout",{"restore_cache":{"keys":["kubeconform-{{ .Environment.KUBECONFORM_VERSION }}"]}},{"run":{"command":"make kubeconform_test","name":"run k8s validation"}},{"save_cache":{"key":"kubeconform-{{ .Environment.KUBECONFORM_VERSION }}","paths":["./kubeconform"]}}]},"run_opa_test":{"docker":[{"image":"circleci/golang:1.14"}],"environment":null,"steps":["checkout",{"restore_cache":{"keys":["go-mod-v4-{{ checksum \"go.sum\" }}"]}},{"run":{"command":"make lint","name":"Run Lint"}},{"run":{"command":"make opa_test","name":"Run Tests"}},{"save_cache":{"key":"go-mod-v4-{{ checksum \"go.sum\" }}","paths":["./opa"]}}]},"run_terraform_validate":{"docker":[{"image":"circleci/golang:1.14"}],"environment":{"TERRAFORM_VERSION":"\u003c\u003c pipeline.parameters.terraform-version \u003e\u003e"},"steps":["checkout",{"restore_cache":{"keys":["terraform-{{ .Environment.TERRAFORM_VERSION }}"]}},{"run":{"command":"make terraform_test","name":"run terraform validate"}},{"save_cache":{"key":"terraform-{{ .Environment.TERRAFORM_VERSION }}","paths":["./terraform"]}}]}},"parameters":{"kubeconform-version":{"default":"v0.4.2","type":"string"},"terraform-version":{"default":"0.14.6","type":"string"}},"version":2.1,"workflows":{"build_and_push":{"jobs":["run_cspell","run_kubeconform","run_terraform_validate","run_opa_test"]},"version":2}},"default_decision":"/schemas/terraform/aws/deny","ecosystems":{"aws":{"SNYK_CC_TF_1":{"description":"## Overview\nUsing Terraform, the `aws_security_group` resource is used to restrict networking to and from different resources.\nWhen the ingress \"cidr_blocks\" is set to [\"0.0.0.0/0\"] or [\"::/0\"] potentially meaning everyone can access your resource.\n\n## Remediation\nThe aws_security_group ingress.cidr_block property should be populated with a specific IP range or address.\n\n## References\nad\n\n","id":"101","impact":"That potentially everyone can access your resource","issue":"That inbound traffic is allowed to a resource from any source instead of a restricted range","policyEngineType":"opa","publicId":"SNYK-CC-TF-1","references":[],"resolve":"Updating the `cidr_block` attribute with a more restrictive IP range or a specific IP address to ensure traffic can only come from known sources. ","severity":"medium","subType":"Security Group","title":"Security Group allows open ingress","type":"terraform"}}},"ignores":["docker-compose.*","*_test.yaml","cspell.json","*_test.rego"],"prepared_queries":{"kubernetes":"k8s_res := data.schemas.kubernetes.deny","terraform":"tf_res := data.schemas.terraform.deny"}} diff --git a/test/smoke/.iac-data/tf_policy.wasm b/test/smoke/.iac-data/tf_policy.wasm new file mode 100644 index 0000000000..05525a2ecf Binary files /dev/null and b/test/smoke/.iac-data/tf_policy.wasm differ diff --git a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh b/test/smoke/spec/iac/snyk_test_local_exec_spec.sh index e3790c2731..966b5c1380 100644 --- a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh +++ b/test/smoke/spec/iac/snyk_test_local_exec_spec.sh @@ -48,4 +48,50 @@ Describe "Snyk iac test --experimental command" The result of function check_valid_json should be success End End + + Describe "terraform single file scan" + Skip if "execute only in regression test" check_if_regression_test + It "finds issues in terraform file" + When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental + The status should be failure # issues found + The output should include "Testing ../fixtures/iac/terraform/sg_open_ssh.tf..." + + # Outputs issues + The output should include "Infrastructure as code issues:" + The output should include "✗ Security Group allows open ingress [Medium Severity] [SNYK-CC-TF-1] in Security Group" + The output should include " introduced by resource > aws_security_group[allow_ssh] > ingress" + End + + It "filters out issues when using severity threshold" + When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --severity-threshold=high + The status should be success # no issues found + The output should include "Testing ../fixtures/iac/terraform/sg_open_ssh.tf..." + + The output should include "Infrastructure as code issues:" + The output should include "Tested ../fixtures/iac/terraform/sg_open_ssh.tf for known issues, found 0 issues" + End + + # TODO: currently skipped because the parser we're using doesn't fail on invalid terraform + # will be fixed before beta + xIt "outputs an error for invalid terraforom files" + When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh_invalid_hcl2.tf --experimental + The status should be failure + The output should include "Invalid Terraform File!" + End + + It "outputs the expected text when running with --sarif flag" + When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --sarif + The status should be failure + The output should include '"id": "SNYK-CC-TF-1",' + The output should include '"ruleId": "SNYK-CC-TF-1",' + End + + It "outputs the expected text when running with --json flag" + When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --json + The status should be failure + The output should include '"id": "SNYK-CC-TF-1",' + The output should include '"packageManager": "terraformconfig",' + The result of function check_valid_json should be success + End + End End