Merge pull request #2029 from snyk/feat/poetry

Feat: Poetry support for @snyk/fix

packages/snyk-fix/package.json

@@ -41,13 +41,15 @@
   "dependencies": {
     "@snyk/dep-graph": "^1.21.0",
     "@snyk/fix-pipenv-pipfile": "0.5.3",
+    "@snyk/fix-poetry": "0.6.0",
     "chalk": "4.1.1",
     "debug": "^4.3.1",
     "lodash.groupby": "4.6.0",
     "lodash.sortby": "^4.7.0",
     "ora": "5.4.0",
     "p-map": "^4.0.0",
-    "strip-ansi": "6.0.0"
+    "strip-ansi": "6.0.0",
+    "toml": "3.0.0"
   },
   "devDependencies": {
     "@types/jest": "26.0.20",

packages/snyk-fix/src/plugins/load-plugin.ts

@@ -7,6 +7,9 @@ export function loadPlugin(type: string): FixHandler {
     case 'pip': {
       return pythonFix;
     }
+    case 'poetry': {
+      return pythonFix;
+    }
     default: {
       throw new UnsupportedTypeError(type);
     }

packages/snyk-fix/src/plugins/package-tool-supported.ts

@@ -1,6 +1,7 @@
 import * as chalk from 'chalk';
 
 import * as pipenvPipfileFix from '@snyk/fix-pipenv-pipfile';
+import * as poetryFix from '@snyk/fix-poetry';
 
 import * as ora from 'ora';
 
@@ -12,10 +13,15 @@ const supportFunc = {
     isSupportedVersion: (version) =>
       pipenvPipfileFix.isPipenvSupportedVersion(version),
   },
+  poetry: {
+    isInstalled: () => poetryFix.isPoetryInstalled(),
+    isSupportedVersion: (version) =>
+      poetryFix.isPoetrySupportedVersion(version),
+  },
 };
 
 export async function checkPackageToolSupported(
-  packageManager: 'pipenv',
+  packageManager: 'pipenv' | 'poetry',
   options: FixOptions,
 ): Promise<void> {
   const { version } = await supportFunc[packageManager].isInstalled();

packages/snyk-fix/src/plugins/python/get-handler-type.ts

@@ -15,6 +15,8 @@ export function getHandlerType(
     return SUPPORTED_HANDLER_TYPES.REQUIREMENTS;
   } else if (['Pipfile'].includes(path.base)) {
     return SUPPORTED_HANDLER_TYPES.PIPFILE;
+  } else if (['pyproject.toml', 'poetry.lock'].includes(path.base)) {
+    return SUPPORTED_HANDLER_TYPES.POETRY;
   }
   return null;
 }

packages/snyk-fix/src/plugins/python/handlers/poetry/index.ts

@@ -0,0 +1,42 @@
+import * as debugLib from 'debug';
+import * as ora from 'ora';
+
+import { EntityToFix, FixOptions } from '../../../../types';
+import { checkPackageToolSupported } from '../../../package-tool-supported';
+import { PluginFixResponse } from '../../../types';
+import { updateDependencies } from './update-dependencies';
+
+const debug = debugLib('snyk-fix:python:Poetry');
+
+export async function poetry(
+  fixable: EntityToFix[],
+  options: FixOptions,
+): Promise<PluginFixResponse> {
+  debug(`Preparing to fix ${fixable.length} Python Poetry projects`);
+  const handlerResult: PluginFixResponse = {
+    succeeded: [],
+    failed: [],
+    skipped: [],
+  };
+
+  await checkPackageToolSupported('poetry', options);
+  for (const [index, entity] of fixable.entries()) {
+    const spinner = ora({ isSilent: options.quiet, stream: process.stdout });
+    const spinnerMessage = `Fixing pyproject.toml ${index + 1}/${
+      fixable.length
+    }`;
+    spinner.text = spinnerMessage;
+    spinner.start();
+
+    const { failed, succeeded, skipped } = await updateDependencies(
+      entity,
+      options,
+    );
+    handlerResult.succeeded.push(...succeeded);
+    handlerResult.failed.push(...failed);
+    handlerResult.skipped.push(...skipped);
+    spinner.stop();
+  }
+
+  return handlerResult;
+}

packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/index.ts

@@ -0,0 +1,175 @@
+import * as pathLib from 'path';
+import * as toml from 'toml';
+
+import * as debugLib from 'debug';
+import * as poetryFix from '@snyk/fix-poetry';
+
+import { PluginFixResponse } from '../../../../types';
+import {
+  DependencyPins,
+  EntityToFix,
+  FixChangesSummary,
+  FixOptions,
+} from '../../../../../types';
+import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied';
+import { CommandFailedError } from '../../../../../lib/errors/command-failed-to-run-error';
+import { validateRequiredData } from '../../validate-required-data';
+import { standardizePackageName } from '../../../standardize-package-name';
+
+const debug = debugLib('snyk-fix:python:Poetry');
+
+interface PyProjectToml {
+  tool: {
+    poetry: {
+      name: string;
+      version: string;
+      description: string;
+      authors: string[];
+      dependencies?: object;
+      'dev-dependencies'?: object;
+    };
+  };
+}
+export async function updateDependencies(
+  entity: EntityToFix,
+  options: FixOptions,
+): Promise<PluginFixResponse> {
+  const handlerResult: PluginFixResponse = {
+    succeeded: [],
+    failed: [],
+    skipped: [],
+  };
+  let poetryCommand;
+  try {
+    const { upgrades, devUpgrades } = await generateUpgrades(entity);
+    const { remediation, targetFile } = validateRequiredData(entity);
+    const targetFilePath = pathLib.resolve(entity.workspace.path, targetFile);
+    const { dir } = pathLib.parse(targetFilePath);
+    // TODO: for better support we need to:
+    // 1. parse the manifest and extract original requirements, version spec etc
+    // 2. swap out only the version and retain original spec
+    // 3. re-lock the lockfile
+
+    // update prod dependencies first
+    if (!options.dryRun && upgrades.length) {
+      const res = await poetryFix.poetryAdd(dir, upgrades, {});
+      if (res.exitCode !== 0) {
+        poetryCommand = res.command;
+        throwPoetryError(res.stderr ? res.stderr : res.stdout, res.command);
+      }
+    }
+
+    // update dev dependencies second
+    if (!options.dryRun && devUpgrades.length) {
+      const res = await poetryFix.poetryAdd(dir, devUpgrades, {
+        dev: true,
+      });
+      if (res.exitCode !== 0) {
+        poetryCommand = res.command;
+        throwPoetryError(res.stderr ? res.stderr : res.stdout, res.command);
+      }
+    }
+    const changes = generateSuccessfulChanges(remediation.pin);
+    handlerResult.succeeded.push({ original: entity, changes });
+  } catch (error) {
+    debug(
+      `Failed to fix ${entity.scanResult.identity.targetFile}.\nERROR: ${error}`,
+    );
+    handlerResult.failed.push({
+      original: entity,
+      error,
+      tip: poetryCommand ? `Try running \`${poetryCommand}\`` : undefined,
+    });
+  }
+  return handlerResult;
+}
+
+export function generateSuccessfulChanges(
+  pins: DependencyPins,
+): FixChangesSummary[] {
+  const changes: FixChangesSummary[] = [];
+  for (const pkgAtVersion of Object.keys(pins)) {
+    const pin = pins[pkgAtVersion];
+    const updatedMessage = pin.isTransitive ? 'Pinned' : 'Upgraded';
+    const newVersion = pin.upgradeTo.split('@')[1];
+    const [pkgName, version] = pkgAtVersion.split('@');
+
+    changes.push({
+      success: true,
+      userMessage: `${updatedMessage} ${pkgName} from ${version} to ${newVersion}`,
+      issueIds: pin.vulns,
+      from: pkgAtVersion,
+      to: `${pkgName}@${newVersion}`,
+    });
+  }
+  return changes;
+}
+
+export async function generateUpgrades(
+  entity: EntityToFix,
+): Promise<{ upgrades: string[]; devUpgrades: string[] }> {
+  const { remediation, targetFile } = validateRequiredData(entity);
+  const pins = remediation.pin;
+
+  const targetFilePath = pathLib.resolve(entity.workspace.path, targetFile);
+  const { dir } = pathLib.parse(targetFilePath);
+  const pyProjectTomlRaw = await entity.workspace.readFile(
+    pathLib.resolve(dir, 'pyproject.toml'),
+  );
+  const pyProjectToml: PyProjectToml = toml.parse(pyProjectTomlRaw);
+
+  const prodTopLevelDeps = Object.keys(
+    pyProjectToml.tool.poetry.dependencies ?? {},
+  );
+  const devTopLevelDeps = Object.keys(
+    pyProjectToml.tool.poetry['dev-dependencies'] ?? {},
+  );
+
+  const upgrades: string[] = [];
+  const devUpgrades: string[] = [];
+  for (const pkgAtVersion of Object.keys(pins)) {
+    const pin = pins[pkgAtVersion];
+    const newVersion = pin.upgradeTo.split('@')[1];
+    const [pkgName] = pkgAtVersion.split('@');
+
+    const upgrade = `${standardizePackageName(pkgName)}==${newVersion}`;
+
+    if (pin.isTransitive) {
+      // transitive and it could have come from a dev or prod dep
+      // since we can't tell right now let be pinned into production deps
+      upgrades.push(upgrade);
+    } else if (prodTopLevelDeps.includes(pkgName)) {
+      upgrades.push(upgrade);
+    } else if (entity.options.dev && devTopLevelDeps.includes(pkgName)) {
+      devUpgrades.push(upgrade);
+    } else {
+      debug(
+        `Could not determine what type of upgrade ${upgrade} is. When choosing between: transitive upgrade, production or dev direct upgrade. `,
+      );
+    }
+  }
+  return { upgrades, devUpgrades };
+}
+
+function throwPoetryError(stderr: string, command?: string) {
+  const ALREADY_UP_TO_DATE = 'No dependencies to install or update';
+  const INCOMPATIBLE_PYTHON = new RegExp(
+    /Python requirement (.*) is not compatible/g,
+    'gm',
+  );
+  const match = INCOMPATIBLE_PYTHON.exec(stderr);
+  if (match) {
+    throw new CommandFailedError(
+      `The current project's Python requirement ${match[1]} is not compatible with some of the required packages`,
+      command,
+    );
+  }
+  // TODO: test this
+  if (stderr.includes(ALREADY_UP_TO_DATE)) {
+    throw new CommandFailedError(
+      'No dependencies could be updated as they seem to be at the correct versions. Make sure installed dependencies in the environment match those in the lockfile by running `poetry update`',
+      command,
+    );
+  }
+  throw new NoFixesCouldBeAppliedError();
+}

packages/snyk-fix/src/plugins/python/load-handler.ts

@@ -1,5 +1,6 @@
 import { pipRequirementsTxt } from './handlers/pip-requirements';
 import { pipenvPipfile } from './handlers/pipenv-pipfile';
+import { poetry } from './handlers/poetry';
 
 import { SUPPORTED_HANDLER_TYPES } from './supported-handler-types';
 
@@ -11,6 +12,9 @@ export function loadHandler(type: SUPPORTED_HANDLER_TYPES) {
     case SUPPORTED_HANDLER_TYPES.PIPFILE: {
       return pipenvPipfile;
     }
+    case SUPPORTED_HANDLER_TYPES.POETRY: {
+      return poetry;
+    }
     default: {
       throw new Error('No handler available for requested project type');
     }

packages/snyk-fix/src/plugins/python/map-entities-per-handler-type.ts

@@ -19,6 +19,7 @@ export function mapEntitiesPerHandlerType(
   } = {
     [SUPPORTED_HANDLER_TYPES.REQUIREMENTS]: [],
     [SUPPORTED_HANDLER_TYPES.PIPFILE]: [],
+    [SUPPORTED_HANDLER_TYPES.POETRY]: [],
   };
 
   const skipped: Array<WithUserMessage<EntityToFix>> = [];

packages/snyk-fix/src/plugins/python/supported-handler-types.ts

@@ -2,4 +2,5 @@ export enum SUPPORTED_HANDLER_TYPES {
   // shortname = display name
   REQUIREMENTS = 'requirements.txt',
   PIPFILE = 'Pipfile',
+  POETRY = 'pyproject.toml',
 }

packages/snyk-fix/src/types.ts

@@ -193,6 +193,7 @@ export interface EntityToFix {
 // add more as needed
 export interface PythonTestOptions {
   command?: string; // python interpreter to use for python tests
+  dev?: boolean;
 }
 export type CliTestOptions = PythonTestOptions;
 export interface WithError<Original> {

packages/snyk-fix/test/acceptance/plugins/package-tool-supported.spec.ts

@@ -1,7 +1,10 @@
 import * as pipenvPipfileFix from '@snyk/fix-pipenv-pipfile';
+import * as poetryFix from '@snyk/fix-poetry';
+
 import { checkPackageToolSupported } from '../../../src/plugins/package-tool-supported';
 
 jest.mock('@snyk/fix-pipenv-pipfile');
+jest.mock('@snyk/fix-poetry');
 
 describe('checkPackageToolSupported', () => {
   it('pipenv fix package called with correct data', async () => {
@@ -28,4 +31,28 @@ describe('checkPackageToolSupported', () => {
     expect(isPipenvSupportedVersionSpy).toHaveBeenCalled();
     expect(isPipenvSupportedVersionSpy).toHaveBeenCalledWith('123.123.123');
   });
+  it('poetry fix package called with correct data', async () => {
+    // Arrange
+    const isPipenvSupportedVersionSpy = jest
+      .spyOn(poetryFix, 'isPoetrySupportedVersion')
+      .mockReturnValue({
+        supported: true,
+        versions: ['1', '2'],
+      });
+    const isPipenvInstalledSpy = jest
+      .spyOn(poetryFix, 'isPoetryInstalled')
+      .mockResolvedValue({
+        version: '3',
+      });
+
+    // Act
+    await checkPackageToolSupported('poetry', {});
+
+    // Assert
+    expect(isPipenvInstalledSpy).toHaveBeenCalled();
+    expect(isPipenvInstalledSpy).toHaveBeenNthCalledWith(1);
+
+    expect(isPipenvSupportedVersionSpy).toHaveBeenCalled();
+    expect(isPipenvSupportedVersionSpy).toHaveBeenCalledWith('3');
+  });
 });