Merge pull request #2087 from snyk/chore/async-test-with-polling

chore: init support async test with polling

src/lib/common.ts

@@ -0,0 +1,3 @@
+export async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

src/lib/ecosystems/polling.ts

@@ -0,0 +1,129 @@
+import * as config from '../config';
+import { isCI } from '../is-ci';
+import { makeRequest } from '../request/promise';
+import { Options } from '../types';
+
+import { assembleQueryString } from '../snyk-test/common';
+import { getAuthHeader } from '../api-token';
+import { ScanResult } from './types';
+import { TestDependenciesResult, TestDepGraphMeta } from '../snyk-test/legacy';
+import { sleep } from '../common';
+
+type ResolveAndTestFactsStatus =
+  | 'CANCELLED'
+  | 'ERROR'
+  | 'PENDING'
+  | 'RUNNING'
+  | 'OK';
+
+interface PollingTask {
+  pollInterval: number;
+  maxAttempts: number;
+}
+
+interface ResolveAndTestFactsResponse {
+  token: string;
+  pollingTask: PollingTask;
+  result?: TestDependenciesResult;
+  meta?: TestDepGraphMeta;
+  status?: ResolveAndTestFactsStatus;
+  code?: number;
+  error?: string;
+  message?: string;
+  userMessage?: string;
+}
+
+export async function requestPollingToken(
+  options: Options,
+  isAsync: boolean,
+  scanResult: ScanResult,
+): Promise<ResolveAndTestFactsResponse> {
+  const payload = {
+    method: 'POST',
+    url: `${config.API}/test-dependencies`,
+    json: true,
+    headers: {
+      'x-is-ci': isCI(),
+      authorization: getAuthHeader(),
+    },
+    body: {
+      isAsync,
+      scanResult,
+    },
+    qs: assembleQueryString(options),
+  };
+  const response = await makeRequest<ResolveAndTestFactsResponse>(payload);
+  throwIfRequestPollingTokenFailed(response);
+  return response;
+}
+
+function throwIfRequestPollingTokenFailed(res: ResolveAndTestFactsResponse) {
+  const { token, status, pollingTask } = res;
+  const { maxAttempts, pollInterval } = pollingTask;
+  const isMissingPollingTask = !!maxAttempts && !!pollInterval;
+  if (!token && !status && isMissingPollingTask) {
+    throw 'Something went wrong, invalid response.';
+  }
+}
+
+export async function pollingWithTokenUntilDone(
+  token: string,
+  type: string,
+  options: Options,
+  pollInterval: number,
+  attemptsCount: number,
+  maxAttempts = Infinity,
+): Promise<ResolveAndTestFactsResponse> {
+  const payload = {
+    method: 'GET',
+    url: `${config.API}/test-dependencies/${token}`,
+    json: true,
+    headers: {
+      'x-is-ci': isCI(),
+      authorization: getAuthHeader(),
+    },
+    qs: { ...assembleQueryString(options), type },
+  };
+
+  const response = await makeRequest<ResolveAndTestFactsResponse>(payload);
+
+  if (pollingRequestHasFailed(response)) {
+    throw response;
+  }
+
+  const taskCompleted = response.result && response.meta;
+  if (taskCompleted) {
+    return response;
+  }
+
+  attemptsCount++;
+  checkPollingAttempts(maxAttempts)(attemptsCount);
+
+  await sleep(pollInterval);
+
+  return await pollingWithTokenUntilDone(
+    token,
+    type,
+    options,
+    pollInterval,
+    attemptsCount,
+    maxAttempts,
+  );
+}
+
+function checkPollingAttempts(maxAttempts: number) {
+  return (attemptsCount: number) => {
+    if (attemptsCount > maxAttempts) {
+      throw new Error('Exceeded Polling maxAttempts');
+    }
+  };
+}
+
+function pollingRequestHasFailed(
+  response: ResolveAndTestFactsResponse,
+): boolean {
+  const { token, result, meta, status, error, code, message } = response;
+  const hasError = !!error && !!code && !!message;
+  const pollingContextIsMissing = !token && !result && !meta && !status;
+  return !!pollingContextIsMissing || hasError;
+}

src/lib/ecosystems/resolve-test-facts.ts

@@ -0,0 +1,49 @@
+import { Options } from '../types';
+import * as spinner from '../../lib/spinner';
+import { Ecosystem, ScanResult, TestResult } from './types';
+import { pollingWithTokenUntilDone, requestPollingToken } from './polling';
+
+export async function resolveAndTestFacts(
+  ecosystem: Ecosystem,
+  scans: {
+    [dir: string]: ScanResult[];
+  },
+  options: Options,
+): Promise<[TestResult[], string[]]> {
+  const results: any[] = [];
+  const errors: string[] = [];
+
+  for (const [path, scanResults] of Object.entries(scans)) {
+    await spinner(`Resolving and Testing fileSignatures in ${path}`);
+    for (const scanResult of scanResults) {
+      try {
+        const res = await requestPollingToken(options, true, scanResult);
+        const { maxAttempts, pollInterval } = res.pollingTask;
+        const attemptsCount = 0;
+        const response = await pollingWithTokenUntilDone(
+          res.token,
+          ecosystem,
+          options,
+          pollInterval,
+          attemptsCount,
+          maxAttempts,
+        );
+        results.push({
+          issues: response?.result?.issues,
+          issuesData: response?.result?.issuesData,
+          depGraphData: response?.result?.depGraphData,
+        });
+      } catch (error) {
+        const hasStatusCodeError = error.code >= 400 && error.code <= 500;
+        if (hasStatusCodeError) {
+          errors.push(error.message);
+          continue;
+        }
+        const failedPath = path ? `in ${path}` : '.';
+        errors.push(`Could not test dependencies ${failedPath}`);
+      }
+    }
+  }
+  spinner.clearAll();
+  return [results, errors];
+}

src/lib/snyk-test/legacy.ts

@@ -236,7 +236,7 @@ interface Issue {
   fixInfo: FixInfo;
 }
 
-interface TestDependenciesResult {
+export interface TestDependenciesResult {
   issuesData: {
     [issueId: string]: IssueData;
   };

test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts

@@ -0,0 +1,36 @@
+export const depGraphData = {
+    schemaVersion: '1.2.0',
+    pkgManager: { name: 'cpp' },
+    pkgs: [
+      { id: '_root@0.0.0', info: { name: '_root', version: '0.0.0' } },
+      {
+        id: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
+        info: {
+          name: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip',
+          version: '0.5.0',
+        },
+      },
+    ],
+    graph: {
+      rootNodeId: 'root-node',
+      nodes: [
+        {
+          nodeId: 'root-node',
+          pkgId: '_root@0.0.0',
+          deps: [
+            {
+              nodeId:
+                'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
+            },
+          ],
+        },
+        {
+          nodeId:
+            'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
+          pkgId: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
+          deps: [],
+        },
+      ],
+    },
+  };
+

test/jest/unit/lib/ecosystems/fixtures/index.ts

@@ -0,0 +1,4 @@
+import { depGraphData } from './depgraph-data';
+import { scanResults } from './scan-results';
+
+export { depGraphData, scanResults };

test/jest/unit/lib/ecosystems/fixtures/scan-results.ts

@@ -0,0 +1,35 @@
+/* eslint-disable @typescript-eslint/camelcase */
+export const scanResults = {
+    path: [
+      {
+        name: 'my-unmanaged-c-project',
+        facts: [
+          {
+            type: 'fileSignatures',
+            data: [
+              {
+                path: 'fastlz_example/fastlz.h',
+                hashes_ffm: [
+                  {
+                    format: 1,
+                    data: 'ucMc383nMM/wkFRM4iOo5Q',
+                  },
+                  {
+                    format: 1,
+                    data: 'k+DxEmslFQWuJsZFXvSoYw',
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+        identity: {
+          type: 'cpp',
+        },
+        target: {
+          remoteUrl: 'https://github.com/some-org/some-unmanaged-project.git',
+          branch: 'master',
+        },
+      },
+    ],
+  };

test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts

@@ -0,0 +1,86 @@
+import { Options } from '../../../../../src/lib/types';
+import * as polling from '../../../../../src/lib/ecosystems/polling';
+import { depGraphData, scanResults } from './fixtures/';
+import { resolveAndTestFacts } from '../../../../../src/lib/ecosystems/resolve-test-facts';
+
+describe('resolve and test facts', () => {
+  afterEach(() => jest.restoreAllMocks());
+
+  const token =
+    'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjlkNGQyMzg0LWUwMmYtNGZiYS1hNWI1LTRhMjU4MzFlM2JmOCIsInNhcGlVcmwiOiJodHRwOi8vd3d3LmZha2Utc2FwaS11cmwvIiwic3RhcnRUaW1lIjoxNjI2MDk2MTg5NzQ1fQ.fyI15bzeB_HtMvqRIBQdKpKBZgQADwn3sByEk64DzxA';
+
+  const pollingTask = {
+    pollInterval: 30000,
+    maxAttempts: 25,
+  };
+
+  it('failing to resolve and test file-signatures fact for c/c++ projects', async () => {
+    const requestPollingTokenSpy = jest.spyOn(polling, 'requestPollingToken');
+    const pollingWithTokenUntilDoneSpy = jest.spyOn(
+      polling,
+      'pollingWithTokenUntilDone',
+    );
+
+    requestPollingTokenSpy.mockResolvedValueOnce({
+      token,
+      status: 'ERROR',
+      pollingTask,
+    });
+
+    pollingWithTokenUntilDoneSpy.mockRejectedValueOnce({
+      code: 500,
+      message:
+        'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)',
+    });
+
+    const [testResults, errors] = await resolveAndTestFacts(
+      'cpp',
+      scanResults,
+      {} as Options,
+    );
+
+    expect(testResults).toEqual([]);
+    expect(errors[0]).toContain(
+      'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)',
+    );
+  });
+
+  it('successfully resolving and testing file-signatures fact for c/c++ projects', async () => {
+    const resolveAndTestFactsSpy = jest.spyOn(polling, 'requestPollingToken');
+    const pollingWithTokenUntilDoneSpy = jest.spyOn(
+      polling,
+      'pollingWithTokenUntilDone',
+    );
+
+    resolveAndTestFactsSpy.mockResolvedValueOnce({
+      token,
+      status: 'OK',
+      pollingTask,
+    });
+
+    pollingWithTokenUntilDoneSpy.mockResolvedValueOnce({
+      token,
+      pollingTask,
+      result: {
+        issuesData: {},
+        issues: [],
+        depGraphData,
+      },
+    });
+
+    const [testResults, errors] = await resolveAndTestFacts(
+      'cpp',
+      scanResults,
+      {} as Options,
+    );
+
+    expect(testResults).toEqual([
+      {
+        issuesData: {},
+        issues: [],
+        depGraphData,
+      },
+    ]);
+    expect(errors).toEqual([]);
+  });
+});