From d21cff7d7b100b4b9d999d5dfefda023d1780eb5 Mon Sep 17 00:00:00 2001 From: anthogez Date: Wed, 14 Jul 2021 15:14:07 +0300 Subject: [PATCH] chore: init support async test with polling Initial support for async test of unmanaged ecosystems e.g. c/c++ adding polling to resolve and test fileSignatures fact. The pr that will introduce the active usage of it will come later on. These changes comes from https://github.com/snyk/snyk/pull/2025 Using chore as prefix, because at the moment it's not bringing any effective feat yet. --- src/lib/common.ts | 3 + src/lib/ecosystems/polling.ts | 129 ++++++++++++++++++ src/lib/ecosystems/resolve-test-facts.ts | 49 +++++++ src/lib/snyk-test/legacy.ts | 2 +- .../lib/ecosystems/fixtures/depgraph-data.ts | 36 +++++ .../unit/lib/ecosystems/fixtures/index.ts | 4 + .../lib/ecosystems/fixtures/scan-results.ts | 35 +++++ .../lib/ecosystems/resolve-test-facts.spec.ts | 86 ++++++++++++ 8 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 src/lib/common.ts create mode 100644 src/lib/ecosystems/polling.ts create mode 100644 src/lib/ecosystems/resolve-test-facts.ts create mode 100644 test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts create mode 100644 test/jest/unit/lib/ecosystems/fixtures/index.ts create mode 100644 test/jest/unit/lib/ecosystems/fixtures/scan-results.ts create mode 100644 test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts diff --git a/src/lib/common.ts b/src/lib/common.ts new file mode 100644 index 0000000000..f9a5c4a74a --- /dev/null +++ b/src/lib/common.ts @@ -0,0 +1,3 @@ +export async function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/lib/ecosystems/polling.ts b/src/lib/ecosystems/polling.ts new file mode 100644 index 0000000000..0695e79e25 --- /dev/null +++ b/src/lib/ecosystems/polling.ts @@ -0,0 +1,129 @@ +import * as config from '../config'; +import { isCI } from '../is-ci'; +import { makeRequest } from '../request/promise'; +import { Options } from '../types'; + +import { assembleQueryString } from '../snyk-test/common'; +import { getAuthHeader } from '../api-token'; +import { ScanResult } from './types'; +import { TestDependenciesResult, TestDepGraphMeta } from '../snyk-test/legacy'; +import { sleep } from '../common'; + +type ResolveAndTestFactsStatus = + | 'CANCELLED' + | 'ERROR' + | 'PENDING' + | 'RUNNING' + | 'OK'; + +interface PollingTask { + pollInterval: number; + maxAttempts: number; +} + +interface ResolveAndTestFactsResponse { + token: string; + pollingTask: PollingTask; + result?: TestDependenciesResult; + meta?: TestDepGraphMeta; + status?: ResolveAndTestFactsStatus; + code?: number; + error?: string; + message?: string; + userMessage?: string; +} + +export async function requestPollingToken( + options: Options, + isAsync: boolean, + scanResult: ScanResult, +): Promise { + const payload = { + method: 'POST', + url: `${config.API}/test-dependencies`, + json: true, + headers: { + 'x-is-ci': isCI(), + authorization: getAuthHeader(), + }, + body: { + isAsync, + scanResult, + }, + qs: assembleQueryString(options), + }; + const response = await makeRequest(payload); + throwIfRequestPollingTokenFailed(response); + return response; +} + +function throwIfRequestPollingTokenFailed(res: ResolveAndTestFactsResponse) { + const { token, status, pollingTask } = res; + const { maxAttempts, pollInterval } = pollingTask; + const isMissingPollingTask = !!maxAttempts && !!pollInterval; + if (!token && !status && isMissingPollingTask) { + throw 'Something went wrong, invalid response.'; + } +} + +export async function pollingWithTokenUntilDone( + token: string, + type: string, + options: Options, + pollInterval: number, + attemptsCount: number, + maxAttempts = Infinity, +): Promise { + const payload = { + method: 'GET', + url: `${config.API}/test-dependencies/${token}`, + json: true, + headers: { + 'x-is-ci': isCI(), + authorization: getAuthHeader(), + }, + qs: { ...assembleQueryString(options), type }, + }; + + const response = await makeRequest(payload); + + if (pollingRequestHasFailed(response)) { + throw response; + } + + const taskCompleted = response.result && response.meta; + if (taskCompleted) { + return response; + } + + attemptsCount++; + checkPollingAttempts(maxAttempts)(attemptsCount); + + await sleep(pollInterval); + + return await pollingWithTokenUntilDone( + token, + type, + options, + pollInterval, + attemptsCount, + maxAttempts, + ); +} + +function checkPollingAttempts(maxAttempts: number) { + return (attemptsCount: number) => { + if (attemptsCount > maxAttempts) { + throw new Error('Exceeded Polling maxAttempts'); + } + }; +} + +function pollingRequestHasFailed( + response: ResolveAndTestFactsResponse, +): boolean { + const { token, result, meta, status, error, code, message } = response; + const hasError = !!error && !!code && !!message; + const pollingContextIsMissing = !token && !result && !meta && !status; + return !!pollingContextIsMissing || hasError; +} diff --git a/src/lib/ecosystems/resolve-test-facts.ts b/src/lib/ecosystems/resolve-test-facts.ts new file mode 100644 index 0000000000..0bc6c3e5a8 --- /dev/null +++ b/src/lib/ecosystems/resolve-test-facts.ts @@ -0,0 +1,49 @@ +import { Options } from '../types'; +import * as spinner from '../../lib/spinner'; +import { Ecosystem, ScanResult, TestResult } from './types'; +import { pollingWithTokenUntilDone, requestPollingToken } from './polling'; + +export async function resolveAndTestFacts( + ecosystem: Ecosystem, + scans: { + [dir: string]: ScanResult[]; + }, + options: Options, +): Promise<[TestResult[], string[]]> { + const results: any[] = []; + const errors: string[] = []; + + for (const [path, scanResults] of Object.entries(scans)) { + await spinner(`Resolving and Testing fileSignatures in ${path}`); + for (const scanResult of scanResults) { + try { + const res = await requestPollingToken(options, true, scanResult); + const { maxAttempts, pollInterval } = res.pollingTask; + const attemptsCount = 0; + const response = await pollingWithTokenUntilDone( + res.token, + ecosystem, + options, + pollInterval, + attemptsCount, + maxAttempts, + ); + results.push({ + issues: response?.result?.issues, + issuesData: response?.result?.issuesData, + depGraphData: response?.result?.depGraphData, + }); + } catch (error) { + const hasStatusCodeError = error.code >= 400 && error.code <= 500; + if (hasStatusCodeError) { + errors.push(error.message); + continue; + } + const failedPath = path ? `in ${path}` : '.'; + errors.push(`Could not test dependencies ${failedPath}`); + } + } + } + spinner.clearAll(); + return [results, errors]; +} diff --git a/src/lib/snyk-test/legacy.ts b/src/lib/snyk-test/legacy.ts index 1478ac87bc..bda663b432 100644 --- a/src/lib/snyk-test/legacy.ts +++ b/src/lib/snyk-test/legacy.ts @@ -236,7 +236,7 @@ interface Issue { fixInfo: FixInfo; } -interface TestDependenciesResult { +export interface TestDependenciesResult { issuesData: { [issueId: string]: IssueData; }; diff --git a/test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts b/test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts new file mode 100644 index 0000000000..22ec19d4b3 --- /dev/null +++ b/test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts @@ -0,0 +1,36 @@ +export const depGraphData = { + schemaVersion: '1.2.0', + pkgManager: { name: 'cpp' }, + pkgs: [ + { id: '_root@0.0.0', info: { name: '_root', version: '0.0.0' } }, + { + id: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0', + info: { + name: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip', + version: '0.5.0', + }, + }, + ], + graph: { + rootNodeId: 'root-node', + nodes: [ + { + nodeId: 'root-node', + pkgId: '_root@0.0.0', + deps: [ + { + nodeId: + 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0', + }, + ], + }, + { + nodeId: + 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0', + pkgId: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0', + deps: [], + }, + ], + }, + }; + \ No newline at end of file diff --git a/test/jest/unit/lib/ecosystems/fixtures/index.ts b/test/jest/unit/lib/ecosystems/fixtures/index.ts new file mode 100644 index 0000000000..6acfc49ff4 --- /dev/null +++ b/test/jest/unit/lib/ecosystems/fixtures/index.ts @@ -0,0 +1,4 @@ +import { depGraphData } from './depgraph-data'; +import { scanResults } from './scan-results'; + +export { depGraphData, scanResults }; \ No newline at end of file diff --git a/test/jest/unit/lib/ecosystems/fixtures/scan-results.ts b/test/jest/unit/lib/ecosystems/fixtures/scan-results.ts new file mode 100644 index 0000000000..8aa23907d7 --- /dev/null +++ b/test/jest/unit/lib/ecosystems/fixtures/scan-results.ts @@ -0,0 +1,35 @@ +/* eslint-disable @typescript-eslint/camelcase */ +export const scanResults = { + path: [ + { + name: 'my-unmanaged-c-project', + facts: [ + { + type: 'fileSignatures', + data: [ + { + path: 'fastlz_example/fastlz.h', + hashes_ffm: [ + { + format: 1, + data: 'ucMc383nMM/wkFRM4iOo5Q', + }, + { + format: 1, + data: 'k+DxEmslFQWuJsZFXvSoYw', + }, + ], + }, + ], + }, + ], + identity: { + type: 'cpp', + }, + target: { + remoteUrl: 'https://github.com/some-org/some-unmanaged-project.git', + branch: 'master', + }, + }, + ], + }; \ No newline at end of file diff --git a/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts b/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts new file mode 100644 index 0000000000..7792abdcc6 --- /dev/null +++ b/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts @@ -0,0 +1,86 @@ +import { Options } from '../../../../../src/lib/types'; +import * as polling from '../../../../../src/lib/ecosystems/polling'; +import { depGraphData, scanResults } from './fixtures/'; +import { resolveAndTestFacts } from '../../../../../src/lib/ecosystems/resolve-test-facts'; + +describe('resolve and test facts', () => { + afterEach(() => jest.restoreAllMocks()); + + const token = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjlkNGQyMzg0LWUwMmYtNGZiYS1hNWI1LTRhMjU4MzFlM2JmOCIsInNhcGlVcmwiOiJodHRwOi8vd3d3LmZha2Utc2FwaS11cmwvIiwic3RhcnRUaW1lIjoxNjI2MDk2MTg5NzQ1fQ.fyI15bzeB_HtMvqRIBQdKpKBZgQADwn3sByEk64DzxA'; + + const pollingTask = { + pollInterval: 30000, + maxAttempts: 25, + }; + + it('failing to resolve and test file-signatures fact for c/c++ projects', async () => { + const requestPollingTokenSpy = jest.spyOn(polling, 'requestPollingToken'); + const pollingWithTokenUntilDoneSpy = jest.spyOn( + polling, + 'pollingWithTokenUntilDone', + ); + + requestPollingTokenSpy.mockResolvedValueOnce({ + token, + status: 'ERROR', + pollingTask, + }); + + pollingWithTokenUntilDoneSpy.mockRejectedValueOnce({ + code: 500, + message: + 'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)', + }); + + const [testResults, errors] = await resolveAndTestFacts( + 'cpp', + scanResults, + {} as Options, + ); + + expect(testResults).toEqual([]); + expect(errors[0]).toContain( + 'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)', + ); + }); + + it('successfully resolving and testing file-signatures fact for c/c++ projects', async () => { + const resolveAndTestFactsSpy = jest.spyOn(polling, 'requestPollingToken'); + const pollingWithTokenUntilDoneSpy = jest.spyOn( + polling, + 'pollingWithTokenUntilDone', + ); + + resolveAndTestFactsSpy.mockResolvedValueOnce({ + token, + status: 'OK', + pollingTask, + }); + + pollingWithTokenUntilDoneSpy.mockResolvedValueOnce({ + token, + pollingTask, + result: { + issuesData: {}, + issues: [], + depGraphData, + }, + }); + + const [testResults, errors] = await resolveAndTestFacts( + 'cpp', + scanResults, + {} as Options, + ); + + expect(testResults).toEqual([ + { + issuesData: {}, + issues: [], + depGraphData, + }, + ]); + expect(errors).toEqual([]); + }); +});