Merge pull request #1738 from snyk/feat/extract-python-provenance

@snyk/fix: Extract requirements.txt version provenance (-r, -c directives)

packages/snyk-fix/src/plugins/python/handlers/pip-requirements/extract-version-provenance.ts

@@ -0,0 +1,51 @@
+import * as path from 'path';
+import * as debugLib from 'debug';
+
+import { containsRequireDirective } from '.';
+import {
+  parseRequirementsFile,
+  Requirement,
+} from './update-dependencies/requirements-file-parser';
+import { Workspace } from '../../../../types';
+
+interface PythonProvenance {
+  [fileName: string]: Requirement[];
+}
+
+const debug = debugLib('snyk-fix:python:extract-version-provenance');
+
+export async function extractProvenance(
+  workspace: Workspace,
+  dir: string,
+  fileName: string,
+  provenance: PythonProvenance = {},
+): Promise<PythonProvenance> {
+  const requirementsTxt = await workspace.readFile(path.join(dir, fileName));
+  provenance = {
+    ...provenance,
+    [fileName]: parseRequirementsFile(requirementsTxt),
+  };
+  const { containsRequire, matches } = await containsRequireDirective(
+    requirementsTxt,
+  );
+  if (containsRequire) {
+    for (const match of matches) {
+      const requiredFilePath = match[2];
+      if (provenance[requiredFilePath]) {
+        debug('Detected recursive require directive, skipping');
+        continue;
+      }
+
+      provenance = {
+        ...provenance,
+        ...(await extractProvenance(
+          workspace,
+          dir,
+          requiredFilePath,
+          provenance,
+        )),
+      };
+    }
+  }
+  return provenance;
+}

packages/snyk-fix/src/types.ts

@@ -176,11 +176,12 @@ export enum SEVERITY {
 
 export type SupportedScanTypes = 'pip';
 
+export interface Workspace {
+  readFile: (path: string) => Promise<string>;
+  writeFile: (path: string, content: string) => Promise<void>;
+}
 export interface EntityToFix {
-  readonly workspace: {
-    readFile: (path: string) => Promise<string>;
-    writeFile: (path: string, content: string) => Promise<void>;
-  };
+  readonly workspace: Workspace;
   readonly scanResult: ScanResult;
   readonly testResult: TestResult;
   // options

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/extract-provenance.spec.ts

@@ -0,0 +1,143 @@
+import * as fs from 'fs';
+import * as pathLib from 'path';
+import { extractProvenance } from '../../../../../src/plugins/python/handlers/pip-requirements/extract-version-provenance';
+import { parseRequirementsFile } from '../../../../../src/plugins/python/handlers/pip-requirements/update-dependencies/requirements-file-parser';
+
+describe('extractProvenance', () => {
+  const workspacesPath = pathLib.resolve(__dirname, 'workspaces');
+  it('can extract and parse 1 required files', async () => {
+    // Arrange
+    const targetFile = pathLib.resolve(workspacesPath, 'with-require/dev.txt');
+
+    const workspace = {
+      readFile: async (path: string) => {
+        return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8');
+      },
+      writeFile: async () => {
+        return;
+      },
+    };
+    const { dir, base } = pathLib.parse(targetFile);
+
+    // Act
+    const result = await extractProvenance(workspace, dir, base);
+    // Assert
+    const baseTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, 'with-require/base.txt'),
+      'utf-8',
+    );
+    const devTxt = fs.readFileSync(targetFile, 'utf-8');
+
+    expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt));
+    expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt));
+  });
+
+  it('can extract and parse 1 required files', async () => {
+    // Arrange
+    const targetFile = pathLib.resolve(
+      workspacesPath,
+      'with-require-folder-up/reqs/requirements.txt',
+    );
+
+    const workspace = {
+      readFile: async (path: string) => {
+        return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8');
+      },
+      writeFile: async () => {
+        return;
+      },
+    };
+    const { dir, base } = pathLib.parse(targetFile);
+
+    // Act
+    const result = await extractProvenance(workspace, dir, base);
+    // Assert
+    const baseTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, 'with-require-folder-up/base.txt'),
+      'utf-8',
+    );
+    const requirementsTxt = fs.readFileSync(targetFile, 'utf-8');
+
+    expect(result['requirements.txt']).toEqual(
+      parseRequirementsFile(requirementsTxt),
+    );
+    expect(result['../base.txt']).toEqual(parseRequirementsFile(baseTxt));
+  });
+  it('can extract and parse all required files with both -r and -c', async () => {
+    // Arrange
+    const folder = 'with-multiple-requires';
+    const targetFile = pathLib.resolve(workspacesPath, `${folder}/dev.txt`);
+
+    const workspace = {
+      readFile: async (path: string) => {
+        return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8');
+      },
+      writeFile: async () => {
+        return;
+      },
+    };
+    const { dir, base } = pathLib.parse(targetFile);
+
+    // Act
+    const result = await extractProvenance(workspace, dir, base);
+    // Assert
+    const baseTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, pathLib.join(folder, 'base.txt')),
+      'utf-8',
+    );
+    const reqsBaseTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, pathLib.join(folder, 'reqs', 'base.txt')),
+      'utf-8',
+    );
+    const devTxt = fs.readFileSync(targetFile, 'utf-8');
+    const constraintsTxt = fs.readFileSync(
+      pathLib.resolve(
+        workspacesPath,
+        pathLib.join(folder, 'reqs', 'constraints.txt'),
+      ),
+      'utf-8',
+    );
+
+    expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt));
+    expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt));
+    expect(result['reqs/base.txt']).toEqual(parseRequirementsFile(reqsBaseTxt));
+    expect(result['reqs/constraints.txt']).toEqual(
+      parseRequirementsFile(constraintsTxt),
+    );
+  });
+
+  it('can extract and parse all required files when -r is recursive', async () => {
+    // Arrange
+    const folder = 'with-recursive-requires';
+    const targetFile = pathLib.resolve(workspacesPath, `${folder}/dev.txt`);
+
+    const workspace = {
+      readFile: async (path: string) => {
+        return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8');
+      },
+      writeFile: async () => {
+        return;
+      },
+    };
+    const { dir, base } = pathLib.parse(targetFile);
+
+    // Act
+    const result = await extractProvenance(workspace, dir, base);
+    // Assert
+    const baseTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, `${folder}/base.txt`),
+      'utf-8',
+    );
+    const devTxt = fs.readFileSync(targetFile, 'utf-8');
+    const constraintsTxt = fs.readFileSync(
+      pathLib.resolve(workspacesPath, `${folder}/constraints.txt`),
+      'utf-8',
+    );
+
+    expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt));
+    expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt));
+    expect(result['constraints.txt']).toEqual(
+      parseRequirementsFile(constraintsTxt),
+    );
+  });
+});

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/base.txt

@@ -0,0 +1 @@
+Jinja2==2.7.2

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/dev.txt

@@ -0,0 +1,4 @@
+-r reqs/base.txt
+-r base.txt
+-c reqs/constraints.txt
+Django==1.6.1

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/base.txt

@@ -0,0 +1 @@
+click>7.0

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/constraints.txt

@@ -0,0 +1 @@
+Django==1.6.7

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/base.txt

@@ -0,0 +1,3 @@
+click>7.0
+# recursive require!
+-r dev.txt

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/constraints.txt

@@ -0,0 +1 @@
+Django==1.6.7

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/dev.txt

@@ -0,0 +1,3 @@
+-r base.txt
+-c constraints.txt
+Django==1.6.1

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/reqs/requirements.txt

@@ -0,0 +1 @@
+-r ../base.txt

packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require/base.txt

@@ -0,0 +1 @@
+click>7.0