diff --git a/.circleci/config.yml b/.circleci/config.yml index 31af5586..c4260a5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,129 +1,67 @@ version: 2.1 orbs: + node: circleci/node@5.1.0 prodsec: snyk/prodsec-orb@1.0 defaults: &defaults + resource_class: medium docker: - - image: node:16 - working_directory: ~/snyk-python-plugin - -commands: - checkout_and_merge: - steps: - - checkout - - run: - name: Checkout master - command: git checkout origin/master - - run: - name: Merge test branch - command: | - git config user.name "CircleCI" - git config user.email "noop" - git merge --no-edit "$CIRCLE_BRANCH" - - attach_workspace: - at: ~/snyk-python-plugin - test_python: - parameters: - pip_version: - type: string - python_version: - type: string - steps: - - checkout_and_merge - - attach_workspace: - at: ~/snyk-python-plugin - - run: - name: Run tests - command: | - apt -qq update - apt -qq install python3-pip python-pip -y &> /dev/null - curl https://pyenv.run | $SHELL - export PATH=$HOME/.pyenv/bin:$PATH - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - export PYTHON_VER_FULL=`pyenv install --list | grep -v 'Available versions' | awk '{$1=$1};1' | grep "^$PYTHON_VER\.[0-9]\+$" | tail -1` - echo $PYTHON_VER_FULL - # Install the specific release of Python if it isn't already installed - pyenv install -s $PYTHON_VER_FULL - export PATH=$HOME/.pyenv/versions/$PYTHON_VER_FULL/bin:$PATH - pyenv shell $PYTHON_VER_FULL - python --version - export PATH=$HOME/.local/bin:$PATH - export LC_ALL=C.UTF-8 - export LANG=C.UTF-8 - python -m pip install --user --quiet -r dev-requirements.txt --disable-pip-version-check - # pipenv installation always bumps pip version to >18.0.0, we manually install the desired pip version again - python -m pip install --user --quiet pip==$PIP_VER - pyenv rehash - npm run test - environment: - PIP_VER: << parameters.pip_version >> - PYTHON_VER: << parameters.python_version >> + - image: cimg/node:19.6.1 jobs: - install: - <<: *defaults - environment: - NODE_ENV: develop # Required because base image sets it to 'production' - steps: - - checkout_and_merge - - run: - name: Use snyk-main npmjs user - command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc - - run: - name: Install Node dependencies - command: npm install - - persist_to_workspace: - root: . - paths: - - node_modules/ lint: <<: *defaults steps: - - checkout_and_merge + - checkout + - node/install-packages: + with-cache: false + override-ci-command: npm install - run: - name: Run linting tasks command: npm run lint + test: <<: *defaults parameters: node_version: type: string - pip_version: - type: string python_version: type: string - docker: - - image: node:<< parameters.node_version >> steps: - - test_python: - pip_version: << parameters.pip_version >> - python_version: << parameters.python_version >> + - checkout + - setup_remote_docker + - run: + name: Run tests + command: | + BUILDKIT_PROGRESS=plain \ + DOCKER_BUILDKIT=1 \ + docker build \ + --build-arg NODE_VERSION=<< parameters.node_version >> \ + --build-arg PYTHON_VERSION=<< parameters.python_version >> \ + -t snyk-python-plugin:integration-tests-<< parameters.python_version >> \ + -f test/Dockerfile . + docker run --rm snyk-python-plugin:integration-tests-<< parameters.python_version >> + build: <<: *defaults steps: - - checkout_and_merge - - run: - name: Use snyk-main npmjs user - command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc - - run: - name: Install dependencies - command: npm install + - checkout + - node/install-packages: + with-cache: false + override-ci-command: npm install - run: - name: Run tests command: npm run build + release: <<: *defaults docker: - image: node:16 steps: - - checkout_and_merge - - run: - name: Install deps - command: npm install + - checkout + - node/install-packages: + with-cache: false + override-ci-command: npm install - run: - name: Build command: npm run build - run: name: Release @@ -139,46 +77,49 @@ workflows: - snyk-bot-slack channel: os-team-managed-alerts - - install: - name: Install - filters: - branches: - ignore: - - master - lint: name: Lint - requires: - - Install filters: branches: ignore: - - master + - main + - build: name: Build - requires: - - Install filters: branches: ignore: - - master + - main + - test: - name: Node << matrix.node_version >>, Python << matrix.python_version >>, Pip << matrix.pip_version >> + name: Node << matrix.node_version >>, Python << matrix.python_version >> requires: - Lint - Build matrix: parameters: - node_version: ['14', '16'] - pip_version: ['10.0.0','18.1.0'] - python_version: ['2.7','3.6', '3.7', '3.8', '3.9'] + node_version: [ + '14', + '16', + '18', + ] + python_version: [ + '3.8', + '3.9', + '3.10', + '3.11', + ] filters: branches: ignore: - - master + - main + - release: name: Release context: nodejs-lib-release + requires: + - Scan repository for secrets filters: branches: only: - - master + - main diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c99e0a66..f392ef87 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -45,4 +45,4 @@ Remember that you're developing for multiple platforms and versions of node, so ## Contributor Agreement -A pull-request will only be considered for merging into the upstream codebase after you have signed our [contributor agreement](https://github.com/snyk/snyk-python-plugin/blob/master/Contributor-Agreement.md), assigning us the rights to the contributed code and granting you a license to use it in return. If you submit a pull request, you will be prompted to review and sign the agreement with one click (we use [CLA assistant](https://cla-assistant.io/)). +A pull-request will only be considered for merging into the upstream codebase after you have signed our [contributor agreement](https://github.com/snyk/snyk-python-plugin/blob/main/Contributor-Agreement.md), assigning us the rights to the contributed code and granting you a license to use it in return. If you submit a pull request, you will be prompted to review and sign the agreement with one click (we use [CLA assistant](https://cla-assistant.io/)). diff --git a/.github/workflows/pr-housekeeping.yml b/.github/workflows/pr-housekeeping.yml new file mode 100644 index 00000000..502dd230 --- /dev/null +++ b/.github/workflows/pr-housekeeping.yml @@ -0,0 +1,13 @@ +on: + schedule: + - cron: '0 0 * * *' # Every day at midnight + workflow_dispatch: + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v7 + with: + stale-pr-message: "Your PR has not had any activity for 60 days. In 7 days I'll close it. Make some activity to remove this." + close-pr-message: "Your PR has now been stale for 7 days. I'm closing it." diff --git a/README.md b/README.md index c7fbaa28..122caa70 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ their corresponding positions in the original manifest file. ## Contributing -[Guide](https://github.com/snyk/snyk-python-plugin/blob/master/.github/CONTRIBUTING.md) +[Guide](https://github.com/snyk/snyk-python-plugin/blob/main/.github/CONTRIBUTING.md) ### Developing and Testing diff --git a/appveyor.yml b/appveyor.yml index 012715da..3d9833b3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ build: off branches: only: - - master + - main init: - git config --global core.autocrlf true diff --git a/package.json b/package.json index b59561a6..1f21b9dc 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,7 @@ "format:check": "prettier --check '{lib,test}/**/*.{js,ts}'", "format": "prettier --write '{lib,test}/**/*.{js,ts}'", "prepare": "npm run build", - "test": "npm run test:pysrc && npm run test:tap && npm run test:jest", - "test:tap": "cross-env TS_NODE_PROJECT=tsconfig-test.json tap --node-arg=-r --node-arg=ts-node/register ./test/**/*.test.{js,ts} -R spec --timeout=900", + "test": "npm run test:pysrc && npm run test:jest", "test:jest": "jest", "test:pysrc": "python -m unittest discover pysrc", "lint": "npm run build-tests && npm run format:check && eslint --cache '{lib,test}/**/*.{js,ts}'" @@ -30,7 +29,6 @@ "tmp": "0.2.1" }, "devDependencies": { - "@snyk/types-tap": "^1.1.0", "@types/jest": "^28.1.3", "@types/node": "^16.11.66", "@types/tmp": "^0.1.0", @@ -44,7 +42,6 @@ "jest-junit": "^10.0.0", "prettier": "^2.7.1", "sinon": "^2.3.2", - "tap": "^12.6.1", "ts-jest": "^28.0.8", "ts-node": "^8.10.2", "typescript": "^4.8.4" diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 00000000..e2b08d90 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,32 @@ +ARG NODE_VERSION +FROM node:${NODE_VERSION} + +ARG DEVUSER=node +USER ${DEVUSER} + +SHELL ["/bin/bash", "-c"] + +WORKDIR /home/${DEVUSER} + +ARG PYTHON_VERSION +ENV PYTHON_VERSION $PYTHON_VERSION + +RUN set -ex \ + && curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \ + && export PATH="$HOME/.pyenv/bin:$PATH" \ + && pyenv update \ + && pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && pyenv rehash + +ENV PATH="/home/${DEVUSER}/.pyenv/shims:${PATH}" +RUN python --version + +COPY --chown=${DEVUSER}:${DEVUSER} . ./ + +RUN npm install + +ENV PATH="/home/${DEVUSER}/.local/bin:${PATH}" +RUN python -m pip install --user --quiet -r dev-requirements.txt --disable-pip-version-check + +CMD ["npm", "run", "test", "--", "--runInBand"] diff --git a/test/_setup.test.ts b/test/_setup.test.ts deleted file mode 100644 index 0f2afd06..00000000 --- a/test/_setup.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { test } from 'tap'; -import { - ensureVirtualenv, - chdirWorkspaces, - activateVirtualenv, - pipInstall, -} from './test-utils'; - -test('install requirements in "pip-app" venv (may take a while)', async (t) => { - chdirWorkspaces('pip-app'); - ensureVirtualenv('pip-app'); - t.teardown(activateVirtualenv('pip-app')); - try { - pipInstall(); - } catch (error) { - t.bailout(error); - } -}); diff --git a/test/fixtures/updated-manifest/requirements.txt b/test/fixtures/updated-manifest/requirements.txt index a52701a5..fad11630 100644 --- a/test/fixtures/updated-manifest/requirements.txt +++ b/test/fixtures/updated-manifest/requirements.txt @@ -2,8 +2,8 @@ Jinja2==2.7.2 Django==2.0.1 python-etcd==0.4.5 Django-Select2==6.0.1 # this version installs with lowercase so it catches a previous bug in pip_resolve.py -irc==16.2 # this has a cyclic dependecy (interanl jaraco.text <==> jaraco.collections) +irc==16.2 # this has a cyclic dependecy (internal jaraco.text <==> jaraco.collections) testtools==\ 2.3.0 # this has a cycle (fixtures ==> testtols); ./packages/prometheus_client-0.6.0 -transitive>=1.1.1 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +transitive>=1.1.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/test/system/inspect-provenance.test.ts b/test/system/inspect-provenance.test.ts deleted file mode 100644 index 127f9479..00000000 --- a/test/system/inspect-provenance.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { test } from 'tap'; -import { chdirWorkspaces, activateVirtualenv } from '../test-utils'; -import * as depGraphLib from '@snyk/dep-graph'; - -import pluginImpl = require('../../lib'); - -const pipAppExpectedDependenciesOnlyProvenance = { - django: { - data: { - name: 'django', - version: '1.6.1', - labels: { provenance: 'requirements.txt:2' }, - }, - msg: 'django looks ok', - }, - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - labels: { provenance: 'requirements.txt:1' }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - labels: { provenance: 'requirements.txt:3' }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - labels: { provenance: 'requirements.txt:4' }, - }, - msg: 'django-select2 looks ok', - }, - irc: { - data: { - name: 'irc', - version: '16.2', - labels: { provenance: 'requirements.txt:5' }, - }, - msg: 'irc ok, even though it has a cyclic dep, yay!', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - labels: { provenance: 'requirements.txt:6' }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -test('inspect --only-provenance', async (t) => { - chdirWorkspaces('pip-app'); - t.teardown(activateVirtualenv('pip-app')); - const result = await pluginImpl.inspect('.', 'requirements.txt', { - args: ['--only-provenance'], - }); - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree(dependencyGraph, 'pip'); - - t.test('plugin', async (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.notOk(plugin.targetFile, 'no targetfile for requirements.txt'); - }); - - t.test('package', async (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pip-app', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - }); - - t.test('package dependencies', async (t) => { - Object.keys(pipAppExpectedDependenciesOnlyProvenance).forEach((depName) => { - t.match( - pkg.dependencies![depName], - pipAppExpectedDependenciesOnlyProvenance[depName].data as any, - pipAppExpectedDependenciesOnlyProvenance[depName].msg as string - ); - }); - }); -}); - -const pipfileExpectedDependenciesOnlyProvenance = { - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - labels: { provenance: 'Pipfile:9' }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - labels: { provenance: 'Pipfile:7' }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - labels: { provenance: 'Pipfile:11' }, - }, - msg: 'django-select2 looks ok', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - labels: { provenance: 'Pipfile:8' }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -test('inspect --only-provenance for Pipfile', async (t) => { - chdirWorkspaces('pipfile-pipapp'); - t.teardown(activateVirtualenv('pip-app')); - const result = await pluginImpl.inspect('.', 'Pipfile', { - args: ['--only-provenance'], - }); - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree(dependencyGraph, 'pip'); - - t.test('plugin', async (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.match(plugin.targetFile, 'Pipfile'); - }); - - t.test('package', async (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pipfile-pipapp', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - }); - - t.notOk(pkg.dependencies!['django'], 'django skipped (editable)'); - - t.test('package dependencies', async (t) => { - Object.keys(pipfileExpectedDependenciesOnlyProvenance).forEach( - (depName) => { - t.match( - pkg.dependencies![depName], - pipfileExpectedDependenciesOnlyProvenance[depName].data as any, - pipfileExpectedDependenciesOnlyProvenance[depName].msg as string - ); - } - ); - }); -}); - -const setupPyAppExpectedDependenciesProvenance = { - django: { - data: { - name: 'django', - version: '1.6.1', - labels: { provenance: 'setup.py:8' }, - }, - msg: 'django looks ok', - }, - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - labels: { provenance: 'setup.py:8' }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - labels: { provenance: 'setup.py:8' }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - labels: { provenance: 'setup.py:8' }, - }, - msg: 'django-select2 looks ok', - }, - irc: { - data: { - name: 'irc', - version: '16.2', - labels: { provenance: 'setup.py:8' }, - }, - msg: 'irc ok, even though it has a cyclic dep, yay!', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - labels: { provenance: 'setup.py:8' }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -test('inspect setup.py', async (t) => { - chdirWorkspaces('setup_py-app'); - t.teardown(activateVirtualenv('pip-app')); - const result = await pluginImpl.inspect('.', 'setup.py', { - args: ['--only-provenance'], - }); - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree(dependencyGraph, 'pip'); - - t.test('plugin', async (t) => { - t.ok(plugin, 'plugin'); - - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.equal(plugin.targetFile, 'setup.py', 'targetfile is setup.py'); - }); - - t.test('package', async (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'test_package', 'name'); - t.equal(pkg.version, '1.0.2', 'version'); - }); - - t.test('package dependencies', async (t) => { - Object.keys(setupPyAppExpectedDependenciesProvenance).forEach((depName) => { - t.match( - pkg.dependencies![depName], - setupPyAppExpectedDependenciesProvenance[depName].data, - setupPyAppExpectedDependenciesProvenance[depName].msg - ); - }); - }); -}); diff --git a/test/system/inspect.spec.ts b/test/system/inspect.spec.ts index 9e39fe31..862a1d33 100644 --- a/test/system/inspect.spec.ts +++ b/test/system/inspect.spec.ts @@ -3,16 +3,37 @@ import { inspect, RequiredPackagesMissingError, } from '../../lib'; -import { chdirWorkspaces } from '../test-utils'; +import * as testUtils from '../test-utils'; +import { chdirWorkspaces, ensureVirtualenv } from '../test-utils'; +import * as depGraphLib from '@snyk/dep-graph'; import { DepGraphBuilder } from '@snyk/dep-graph'; import { FILENAMES } from '../../lib/types'; import * as subProcess from '../../lib/dependencies/sub-process'; import { SpawnSyncReturns } from 'child_process'; -import * as depGraphLib from '@snyk/dep-graph'; import * as fs from 'fs'; import * as path from 'path'; -// TODO: jestify tap tests in ./inspect.test.js here +// Usually the setup of virtual environments can run for a while +jest.setTimeout(120000); + +interface DependencyInfo { + pkg: depGraphLib.Pkg; + directDeps: string[]; +} + +// We can't do a full dependency graph comparison, as generated dependency graphs vary wildly +// between Python versions. Instead, we ensure that the transitive lines are not broken. +function compareTransitiveLines( + received: depGraphLib.DepGraph, + expected: DependencyInfo[] +) { + expected.forEach((depInfo: DependencyInfo) => { + expect( + received.directDepsLeadingTo(depInfo.pkg).map((pkg) => pkg.name) + ).toEqual(depInfo.directDeps); + }); +} + describe('inspect', () => { const originalCurrentWorkingDirectory = process.cwd(); @@ -20,10 +41,237 @@ describe('inspect', () => { process.chdir(originalCurrentWorkingDirectory); }); + describe('when doing inspect with --only-provenance', () => { + let tearDown; + beforeAll(() => { + const workspace = 'pip-app'; + chdirWorkspaces(workspace); + ensureVirtualenv(workspace); + tearDown = testUtils.activateVirtualenv(workspace); + testUtils.pipInstall(); + }); + + afterAll(() => { + tearDown(); + }); + + it.each([ + { + workspace: 'pip-app', + targetFile: FILENAMES.pip.manifest, + }, + { + workspace: 'pipfile-pipapp', + targetFile: FILENAMES.pipenv.manifest, + }, + { + workspace: 'setup_py-app', + targetFile: FILENAMES.setuptools.manifest, + }, + ])( + 'should get a valid dependency graph for workspace = $workspace', + async ({ workspace, targetFile }) => { + testUtils.chdirWorkspaces(workspace); + + const result = await inspect('.', targetFile, { + args: ['--only-provenance'], + }); + expect(result.dependencyGraph.toJSON()).not.toEqual({}); + } + ); + }); + + describe('when testing pip projects', () => { + let tearDown; + afterEach(() => { + tearDown(); + }); + + it.each([ + { + workspace: 'pip-app', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'jaraco.collections', + version: '4.3.0', + }, + directDeps: ['irc'], + }, + { + pkg: { + name: 'django-appconf', + version: '1.0.5', + }, + directDeps: ['django-select2'], + }, + ], + }, + { + workspace: 'pip-app-bom', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'markupsafe', + version: '2.1.3', + }, + directDeps: ['jinja2'], + }, + ], + }, + { + workspace: 'pip-app-deps-with-urls', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'markupsafe', + version: '2.1.3', + }, + directDeps: ['jinja2'], + }, + ], + }, + { + workspace: 'pip-app-without-markupsafe', + uninstallPackages: ['MarkupSafe'], + pluginOpts: { allowMissing: true }, + expected: [ + { + pkg: { + name: 'markupsafe', + version: '?', + }, + directDeps: ['jinja2'], + }, + ], + }, + { + workspace: 'pip-app-deps-not-installed', + uninstallPackages: [], + pluginOpts: { allowMissing: true }, + expected: [ + { + pkg: { + name: 's3transfer', + version: '0.6.2', + }, + directDeps: ['awss'], + }, + ], + }, + { + workspace: 'pip-app-trusted-host', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'markupsafe', + version: '2.1.3', + }, + directDeps: ['jinja2'], + }, + ], + }, + { + workspace: 'pip-app-deps-with-dashes', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'dj-database-url', + version: '0.4.2', + }, + directDeps: ['dj-database-url'], + }, + ], + }, + { + workspace: 'pip-app-with-openapi_spec_validator', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'jsonschema', + version: '4.19.0', + }, + directDeps: ['openapi-spec-validator'], + }, + ], + }, + { + workspace: 'pip-app-deps-conditional', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'posix-ipc', + version: '1.0.0', + }, + directDeps: ['posix-ipc'], + }, + ], + }, + { + workspace: 'pip-app-deps-editable', + uninstallPackages: [], + pluginOpts: {}, + expected: [ + { + pkg: { + name: 'posix-ipc', + version: '1.0.0', + }, + directDeps: ['posix-ipc'], + }, + ], + }, + ])( + 'should get a valid dependency graph for workspace = $workspace', + async ({ workspace, uninstallPackages, pluginOpts, expected }) => { + testUtils.chdirWorkspaces(workspace); + testUtils.ensureVirtualenv(workspace); + tearDown = testUtils.activateVirtualenv(workspace); + testUtils.pipInstall(); + if (uninstallPackages) { + uninstallPackages.forEach((pkg) => { + testUtils.pipUninstall(pkg); + }); + } + + const result = await inspect('.', FILENAMES.pip.manifest, pluginOpts); + compareTransitiveLines(result.dependencyGraph, expected); + } + ); + + it('should fail on missing transitive dependencies', async () => { + const workspace = 'pip-app'; + const virtualEnv = 'pip-app-without-markupsafe'; + testUtils.chdirWorkspaces(workspace); + testUtils.ensureVirtualenv(virtualEnv); + tearDown = testUtils.activateVirtualenv(workspace); + testUtils.pipInstall(); + testUtils.pipUninstall('MarkupSafe'); + + await expect( + async () => await inspect('.', FILENAMES.pip.manifest) + ).rejects.toThrow('Required packages missing: markupsafe'); + }); + }); + describe('poetry projects', () => { it('should return expected dependencies for poetry-app', async () => { const workspace = 'poetry-app'; - chdirWorkspaces(workspace); + testUtils.chdirWorkspaces(workspace); const result = await inspect('.', FILENAMES.poetry.lockfile); expect(result).toMatchObject({ @@ -90,6 +338,65 @@ describe('inspect', () => { }); }); + describe('when generating Pipfile depGraphs ', () => { + let tearDown; + beforeAll(() => { + const workspace = 'pip-app'; + testUtils.chdirWorkspaces(workspace); + testUtils.ensureVirtualenv(workspace); + tearDown = testUtils.activateVirtualenv(workspace); + testUtils.pipInstall(); + }); + + afterAll(() => { + tearDown(); + }); + + it.each([ + { + workspace: 'pipfile-pipapp-pinned', + }, + { + workspace: 'pipenv-app', + }, + { + workspace: 'pipfile-pipapp', + targetFile: undefined, + }, + { + workspace: 'pipfile-nested-dirs', + targetFile: 'nested/directory/Pipfile', + }, + ])( + 'should get a valid dependency graph for workspace = $workspace', + async ({ workspace, targetFile }) => { + testUtils.chdirWorkspaces(workspace); + const result = await inspect( + '.', + targetFile ? targetFile : FILENAMES.pipenv.manifest + ); + + const expected = [ + { + pkg: { + name: 'markupsafe', + version: '2.1.3', + }, + directDeps: ['jinja2'], + }, + ]; + compareTransitiveLines(result.dependencyGraph, expected); + } + ); + + it('should fail with no deps or dev-deps', async () => { + testUtils.chdirWorkspaces('pipfile-empty'); + await expect( + async () => await inspect('.', FILENAMES.pipenv.manifest) + ).rejects.toThrow('No dependencies detected in manifest'); + }); + }); + describe('setup.py projects', () => { const mockedExecuteSync = jest.spyOn(subProcess, 'executeSync'); const mockedExecute = jest.spyOn(subProcess, 'execute'); @@ -138,7 +445,7 @@ describe('inspect', () => { mockedExecute.mockClear(); }); - it('should return dep graph for very dence input', async () => { + it('should return dep graph for very dense input', async () => { const manifestFilePath = `test/fixtures/pipenv-project/Pipfile`; const result = await inspect('.', manifestFilePath); @@ -181,7 +488,7 @@ describe('inspect', () => { ); const manifestFilePath = 'path/to/requirements.txt'; - expect(inspect('.', manifestFilePath)).rejects.toThrowError( + await expect(inspect('.', manifestFilePath)).rejects.toThrowError( new EmptyManifestError('No dependencies detected in manifest.') ); }); @@ -193,7 +500,7 @@ describe('inspect', () => { mockedExecute.mockRejectedValueOnce('Required packages missing'); const manifestFilePath = 'path/to/requirements.txt'; - expect(inspect('.', manifestFilePath)).rejects.toThrowError( + await expect(inspect('.', manifestFilePath)).rejects.toThrowError( new RequiredPackagesMissingError( 'Required packages missing\n' + 'Please run `pip install -r path/to/requirements.txt`. If the issue persists try again with --skip-unresolved.' diff --git a/test/system/inspect.test.js b/test/system/inspect.test.js deleted file mode 100644 index f4c2d30c..00000000 --- a/test/system/inspect.test.js +++ /dev/null @@ -1,1270 +0,0 @@ -import * as depGraphLib from '@snyk/dep-graph'; - -const test = require('tap').test; -const fs = require('fs'); -const sinon = require('sinon'); - -const plugin = require('../../lib'); -const subProcess = require('../../lib/dependencies/sub-process'); -const testUtils = require('../test-utils'); -const os = require('os'); - -const chdirWorkspaces = testUtils.chdirWorkspaces; - -function normalize(s = '') { - return s.replace(/\r/g, ''); -} - -const pipAppExpectedDependencies = { - django: { - data: { - name: 'django', - version: '1.6.1', - }, - msg: 'django looks ok', - }, - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - dependencies: { - markupsafe: { - name: 'markupsafe', - version: /.+$/, - }, - }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - dependencies: { - dnspython: { - name: 'dnspython', - version: /.+$/, - }, - urllib3: { - name: 'urllib3', - version: /.+$/, - }, - }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - msg: 'django-select2 looks ok', - }, - irc: { - data: { - name: 'irc', - version: '16.2', - dependencies: { - 'more-itertools': {}, - 'jaraco.functools': {}, - 'jaraco.collections': { - dependencies: { - 'jaraco.text': {}, - }, - }, - 'jaraco.text': { - dependencies: { - 'jaraco.functools': {}, - }, - }, - }, - }, - msg: 'irc ok, even though it has a cyclic dep, yay!', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - dependencies: { - pbr: {}, - extras: {}, - fixtures: {}, - unittest2: {}, - traceback2: {}, - 'python-mimeparse': {}, - }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -const pipfilePinnedExpectedDependencies = { - django: { - data: { - name: 'django', - version: '1.6.1', - }, - msg: 'django looks ok', - }, - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - dependencies: { - markupsafe: { - name: 'markupsafe', - version: /.+$/, - }, - }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - dependencies: { - dnspython: { - name: 'dnspython', - version: /.+$/, - }, - urllib3: { - name: 'urllib3', - version: /.+$/, - }, - }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - msg: 'django-select2 looks ok', - }, - irc: { - data: { - name: 'irc', - version: '16.2', - dependencies: { - 'more-itertools': {}, - 'jaraco.functools': {}, - 'jaraco.collections': { - dependencies: { - 'jaraco.text': {}, - }, - }, - 'jaraco.text': { - dependencies: { - 'jaraco.functools': {}, - }, - }, - }, - }, - msg: 'irc ok, even though it has a cyclic dep, yay!', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - dependencies: { - pbr: {}, - extras: {}, - fixtures: {}, - unittest2: {}, - traceback2: {}, - 'python-mimeparse': {}, - }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -const setupPyAppExpectedDependencies = { - django: { - data: { - name: 'django', - version: '1.6.1', - }, - msg: 'django looks ok', - }, - jinja2: { - data: { - name: 'jinja2', - version: '2.7.2', - dependencies: { - markupsafe: { - name: 'markupsafe', - version: /.+$/, - }, - }, - }, - msg: 'jinja2 looks ok', - }, - 'python-etcd': { - data: { - name: 'python-etcd', - version: '0.4.5', - dependencies: { - dnspython: { - name: 'dnspython', - version: /.+$/, - }, - urllib3: { - name: 'urllib3', - version: /.+$/, - }, - }, - }, - msg: 'python-etcd is ok', - }, - 'django-select2': { - data: { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - msg: 'django-select2 looks ok', - }, - irc: { - data: { - name: 'irc', - version: '16.2', - dependencies: { - 'more-itertools': {}, - 'jaraco.functools': {}, - 'jaraco.collections': { - dependencies: { - 'jaraco.text': {}, - }, - }, - 'jaraco.text': { - dependencies: { - 'jaraco.functools': {}, - }, - }, - }, - }, - msg: 'irc ok, even though it has a cyclic dep, yay!', - }, - testtools: { - data: { - name: 'testtools', - version: '2.3.0', - dependencies: { - pbr: {}, - extras: {}, - fixtures: {}, - unittest2: {}, - traceback2: {}, - 'python-mimeparse': {}, - }, - }, - msg: "testtools ok, even though it's cyclic, yay!", - }, -}; - -test('inspect', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('plugin', (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.notOk(plugin.targetFile, 'no targetfile for requirements.txt'); - t.end(); - }); - - t.test('package', (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pip-app', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - t.end(); - }); - - t.test('package dependencies', (t) => { - Object.keys(pipAppExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - pipAppExpectedDependencies[depName].data, - pipAppExpectedDependencies[depName].msg - ); - }); - - t.end(); - }); - - t.end(); - }); -}); - -test('inspect requirements.txt with bom encoding', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-bom'); - const venvCreated = testUtils.ensureVirtualenv('pip-app-bom'); - t.teardown(testUtils.activateVirtualenv('pip-app-bom')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt').then(async (result) => { - t.ok('has dependencyGraph property', result.dependencyGraph); - t.end(); - }); - }); -}); - -test('inspect setup.py', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('setup_py-app'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'setup.py'); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('plugin', (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.equal(plugin.targetFile, 'setup.py', 'targetfile is setup.py'); - t.end(); - }); - - t.test('package', (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'test_package', 'name'); - t.equal(pkg.version, '1.0.2', 'version'); - t.end(); - }); - - t.test('package dependencies', (t) => { - Object.keys(setupPyAppExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - setupPyAppExpectedDependencies[depName].data, - setupPyAppExpectedDependencies[depName].msg - ); - }); - - t.end(); - }); - t.end(); - }); -}); - -test('inspect setup.py with missing deps', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('setup_py-app'); - t.teardown(testUtils.activateVirtualenv('setup_py-app')); - }) - .then(() => { - return plugin.inspect('.', 'setup.py'); - }) - .catch((error) => { - t.match(normalize(error.message), 'pip install -e .'); - }); -}); - -test('transitive dep not installed', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-without-markupsafe' - ); - t.teardown(testUtils.activateVirtualenv('pip-app-without-markupsafe')); - if (venvCreated) { - testUtils.pipInstall(); - testUtils.pipUninstall('MarkupSafe'); - } - }) - .then(() => { - return plugin - .inspect('.', 'requirements.txt') - .then(() => { - t.fail('should have failed'); - }) - .catch((error) => { - t.equal( - normalize(error.message), - 'Required packages missing: markupsafe\n\nPlease run `pip install -r requirements.txt`. ' + - 'If the issue persists try again with --skip-unresolved.' - ); - t.end(); - }); - }); -}); - -test('transitive dep not installed, but with allowMissing option', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-without-markupsafe' - ); - t.teardown(testUtils.activateVirtualenv('pip-app-without-markupsafe')); - if (venvCreated) { - testUtils.pipInstall(); - testUtils.pipUninstall('MarkupSafe'); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt', { allowMissing: true }); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('plugin', (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.end(); - }); - - t.test('package', (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pip-app', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - t.end(); - }); - - t.test('package dependencies', (t) => { - t.same( - pkg.dependencies.django, - { - name: 'django', - version: '1.6.1', - }, - 'django looks ok' - ); - - t.match( - pkg.dependencies.jinja2, - { - name: 'jinja2', - version: '2.7.2', - dependencies: {}, - }, - 'jinja2 looks ok' - ); - - t.match( - pkg.dependencies['python-etcd'], - { - name: 'python-etcd', - version: '0.4.5', - dependencies: { - dnspython: { - name: 'dnspython', - version: /.+$/, - }, - urllib3: { - name: 'urllib3', - version: /.+$/, - }, - }, - }, - 'python-etcd is ok' - ); - - t.match( - pkg.dependencies['django-select2'], - { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - 'django-select2 looks ok' - ); - - t.end(); - }); - - t.end(); - }); -}); - -test('deps not installed', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-not-installed'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(() => { - t.fail('should have failed'); - }) - .catch((error) => { - t.equal( - normalize(error.message), - 'Required packages missing: awss\n\nPlease run `pip install -r requirements.txt`. ' + - 'If the issue persists try again with --skip-unresolved.' - ); - t.end(); - }); -}); - -test('deps not installed, but with allowMissing option', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-not-installed'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt', { allowMissing: true }); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('plugin', (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.end(); - }); - - t.test('package', (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pip-app-deps-not-installed', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - t.end(); - }); - - t.end(); - }); -}); - -test('uses provided exec command', (t) => { - return Promise.resolve() - .then(() => { - const execute = sinon.stub(subProcess, 'execute'); - execute.onFirstCall().returns(Promise.resolve('abc')); - execute.onSecondCall().returns(Promise.resolve('{}')); - t.teardown(execute.restore); - return execute; - }) - .then((execute) => { - const command = 'echo'; - return plugin - .inspect('.', 'requirements.txt', { command: command }) - .then(() => { - t.ok(execute.calledTwice, 'execute called twice'); - t.equal(execute.firstCall.args[0], command, 'uses command'); - t.equal(execute.secondCall.args[0], command, 'uses command'); - t.end(); - }); - }); -}); - -test('package name differs from requirement (- vs _)', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-with-dashes'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-deps-with-dashes' - ); - t.teardown(testUtils.activateVirtualenv('pip-app-deps-with-dashes')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt', { allowMissing: true }); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - t.same( - pkg.dependencies['dj-database-url'], - { - name: 'dj-database-url', - version: '0.4.2', - }, - 'dj-database-url looks ok' - ); - if (os.platform() !== 'win32') { - t.same( - pkg.dependencies['posix-ipc'], - { - name: 'posix-ipc', - version: '1.0.0', - }, - 'posix-ipc looks ok' - ); - } - t.end(); - }); -}); - -test('package name differs from requirement (- vs .)', (t) => { - t.pass('Not implemented yet'); - t.end(); -}); - -test('package installed conditionally based on python version', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-with-python-markers'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-with-python-markers' - ); - t.teardown(testUtils.activateVirtualenv('pip-app-with-python-markers')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - t.notOk(pkg.dependencies.enum34, 'enum34 dep ignored'); - t.ok(pkg.dependencies.click, 'click dep is present'); - t.end(); - }); -}); - -test('should return correct package info when a single package has a dependency more than once', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-with-repeating-dependency'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-with-repeating-dependency' - ); - t.teardown( - testUtils.activateVirtualenv('pip-app-with-repeating-dependency') - ); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - t.ok(result.dependencyGraph, 'graph generated'); - t.end(); - }); -}); - -test('should work for openapi_spec_validator', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-with-openapi_spec_validator'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-with-openapi_spec_validator' - ); - t.teardown( - testUtils.activateVirtualenv('pip-app-with-openapi_spec_validator') - ); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - t.ok(result.dependencyGraph, 'graph generated'); - t.end(); - }); -}); - -test('Pipfile package found conditionally based on python version', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipfile-markers'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .catch((error) => { - t.match( - normalize(error.message), - 'No dependencies detected in manifest.' - ); - }); -}); - -test('package depends on platform', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-conditional'); - const venvCreated = testUtils.ensureVirtualenv( - 'pip-app-deps-conditional' - ); - t.teardown(testUtils.activateVirtualenv('pip-app-deps-conditional')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt', { allowMissing: true }); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - if (os.platform() !== 'win32') { - t.notOk(pkg.dependencies.pypiwin32, 'win32 dep ignored'); - t.same( - pkg.dependencies['posix-ipc'], - { - name: 'posix-ipc', - version: '1.0.0', - }, - 'posix-ipc looks ok' - ); - } else { - t.ok(pkg.dependencies.pypiwin32, 'win32 installed'); - t.notOk(pkg.dependencies['posix-ipc'], 'not win32 dep skipped'); - } - t.end(); - }); -}); - -test('editables ignored', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-editable'); - const venvCreated = testUtils.ensureVirtualenv('pip-app-deps-editable'); - t.teardown(testUtils.activateVirtualenv('pip-app-deps-editable')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt', { allowMissing: true }); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - t.notOk(pkg.dependencies['simple'], 'editable dep ignored'); - t.notOk(pkg.dependencies['sample'], 'editable subdir dep ignored'); - if (os.platform() !== 'win32') { - t.same( - pkg.dependencies['posix-ipc'], - { - name: 'posix-ipc', - version: '1.0.0', - }, - 'posix-ipc looks ok' - ); - } - t.end(); - }); -}); - -test('deps with options', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-with-options'); - const venvCreated = testUtils.ensureVirtualenv('pip-app-with-options'); - t.teardown(testUtils.activateVirtualenv('pip-app-with-options')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('plugin', (t) => { - t.ok(plugin, 'plugin'); - t.equal(plugin.name, 'snyk-python-plugin', 'name'); - t.match(plugin.runtime, 'Python', 'runtime'); - t.end(); - }); - - t.test('package', (t) => { - t.ok(pkg, 'package'); - t.equal(pkg.name, 'pip-app-with-options', 'name'); - t.equal(pkg.version, '0.0.0', 'version'); - t.end(); - }); - t.test('package dependencies', (t) => { - t.match( - pkg.dependencies.markupsafe, - { - name: 'markupsafe', - version: '1.1.1', - }, - 'MarkupSafe looks ok' - ); - - t.match( - pkg.dependencies.dnspython, - { - name: 'dnspython', - version: '1.13.0', - }, - 'dnspython looks ok' - ); - - t.end(); - }); - - t.end(); - }); -}); - -test('trusted host ignored', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-trusted-host'); - const venvCreated = testUtils.ensureVirtualenv('pip-app-trusted-host'); - t.teardown(testUtils.activateVirtualenv('pip-app-trusted-host')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - t.ok(pkg.dependencies, 'does not error'); - t.end(); - }); -}); - -test('inspect Pipfile', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipfile-pipapp'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.equal(plugin.targetFile, 'Pipfile', 'Pipfile targetfile'); - - t.test('package dependencies', (t) => { - t.notOk(pkg.dependencies['django'], 'django skipped (editable)'); - - t.match( - pkg.dependencies['django-select2'], - { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - 'django-select2 looks ok' - ); - - t.match( - pkg.dependencies['python-etcd'], - { - name: 'python-etcd', - version: /^0\.4.*$/, - }, - 'python-etcd looks ok' - ); - - t.notOk( - pkg.dependencies['e1839a8'], - 'dummy local package skipped (editable)' - ); - - t.ok(pkg.dependencies['jinja2'] !== undefined, 'jinja2 found'); - t.ok(pkg.dependencies['testtools'] !== undefined, 'testtools found'); - - t.end(); - }); - - t.end(); - }); -}); - -test('inspect Pipfile with pinned versions', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipfile-pipapp-pinned'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('package dependencies', (t) => { - Object.keys(pipfilePinnedExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - pipfilePinnedExpectedDependencies[depName].data, - pipfilePinnedExpectedDependencies[depName].msg - ); - }); - - t.end(); - }); - - t.end(); - }); -}); - -const pipenvAppExpectedDependencies = { - 'python-etcd': { - data: { - name: 'python-etcd', - version: /^0\.4/, - }, - msg: 'python-etcd1 found with version >=0.4,<0.5', - }, - jinja2: { - data: { - name: 'jinja2', - version: /^0|1|2|3\.[0-9]/, - }, - msg: 'jinja2 found', - }, - testtools: { - data: { - name: 'testtools', - }, - msg: 'testtools found', - }, -}; - -test('inspect Pipfile in nested directory', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipfile-nested-dirs'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'nested/directory/Pipfile'); - }) - .then(async (result) => { - const plugin = result.plugin; - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.equal( - plugin.targetFile, - 'nested/directory/Pipfile', - 'Pipfile targetfile' - ); - - t.test('package dependencies', (t) => { - t.notOk(pkg.dependencies['django'], 'django skipped (editable)'); - - t.match( - pkg.dependencies['django-select2'], - { - name: 'django-select2', - version: '6.0.1', - dependencies: { - 'django-appconf': { - name: 'django-appconf', - }, - }, - }, - 'django-select2 looks ok' - ); - - t.match( - pkg.dependencies['python-etcd'], - { - name: 'python-etcd', - version: /^0\.4.*$/, - }, - 'python-etcd looks ok' - ); - - t.notOk( - pkg.dependencies['e1839a8'], - 'dummy local package skipped (editable)' - ); - - t.ok(pkg.dependencies['jinja2'] !== undefined, 'jinja2 found'); - t.ok(pkg.dependencies['testtools'] !== undefined, 'testtools found'); - - t.end(); - }); - - t.end(); - }); -}); - -test('package names with urls are skipped', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pip-app-deps-with-urls'); - const venvCreated = testUtils.ensureVirtualenv('pip-app-deps-with-urls'); - t.teardown(testUtils.activateVirtualenv('pip-app-deps-with-urls')); - if (venvCreated) { - testUtils.pipInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'requirements.txt'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - t.equal( - Object.keys(pkg.dependencies).length, - 1, - '1 dependency was skipped' - ); - }); -}); - -test('inspect Pipfile with no deps or dev-deps exits with message', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipfile-empty'); - t.teardown(testUtils.activateVirtualenv('pip-app')); - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .catch((error) => { - t.match( - normalize(error.message), - 'No dependencies detected in manifest.' - ); - }); -}); - -test('inspect pipenv app dev dependencies', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipenv-app'); - - const venvCreated = testUtils.ensureVirtualenv('pipenv-app'); - t.teardown(testUtils.activateVirtualenv('pipenv-app')); - if (venvCreated) { - return testUtils.pipenvInstall({ dev: true }); - } - }) - .then(() => { - return plugin.inspect('.', 'Pipfile', { - dev: true, - }); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('package dependencies', (t) => { - Object.keys(pipenvAppExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - pipenvAppExpectedDependencies[depName].data, - pipenvAppExpectedDependencies[depName].msg - ); - }); - - t.match(pkg.dependencies.bs4, { name: 'bs4' }); - - t.end(); - }); - - t.end(); - }); -}); - -test('inspect pipenv app with auto-created virtualenv', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipenv-app'); - - // Use several teardown callbacks, called in reverse order. - const teardowns = []; - t.teardown(() => { - while (teardowns.length > 0) { - teardowns.pop()(); - } - }); - - if (testUtils.getActiveVenvName() !== null) { - teardowns.push(testUtils.deactivateVirtualenv()); - } - - // Set the WORKON_HOME env var to make pipenv put its auto-created - // virtualenv where we want it. - teardowns.push(testUtils.setWorkonHome()); - - // Have pipenv create and update a virtualenv if it doesn't exist. - const proc = subProcess.executeSync('pipenv', ['--venv']); - if (proc.status !== 0) { - teardowns.push(() => { - fs.unlinkSync('Pipfile.lock'); - }); - const updateProc = subProcess.executeSync('pipenv', ['update']); - if (updateProc.status !== 0) { - t.bailout('Failed to install dependencies using `pipenv update`'); - } - } - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('package dependencies', (t) => { - Object.keys(pipenvAppExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - pipenvAppExpectedDependencies[depName].data, - pipenvAppExpectedDependencies[depName].msg - ); - }); - - t.end(); - }); - - t.end(); - }); -}); - -test('inspect pipenv app with user-created virtualenv', (t) => { - return Promise.resolve() - .then(() => { - chdirWorkspaces('pipenv-app'); - - const venvCreated = testUtils.ensureVirtualenv('pipenv-app'); - t.teardown(testUtils.activateVirtualenv('pipenv-app')); - if (venvCreated) { - return testUtils.pipenvInstall(); - } - }) - .then(() => { - return plugin.inspect('.', 'Pipfile'); - }) - .then(async (result) => { - const dependencyGraph = result.dependencyGraph; - const pkg = await depGraphLib.legacy.graphToDepTree( - dependencyGraph, - 'pip' - ); - - t.test('package dependencies', (t) => { - Object.keys(pipenvAppExpectedDependencies).forEach((depName) => { - t.match( - pkg.dependencies[depName], - pipenvAppExpectedDependencies[depName].data, - pipenvAppExpectedDependencies[depName].msg - ); - }); - - t.end(); - }); - - t.end(); - }); -}); diff --git a/test/test-utils.ts b/test/test-utils.ts index 9d817bf3..cea3452c 100644 --- a/test/test-utils.ts +++ b/test/test-utils.ts @@ -2,7 +2,6 @@ import * as fs from 'fs'; import * as path from 'path'; import * as process from 'process'; - import subProcess = require('../lib/dependencies/sub-process'); export { @@ -12,17 +11,11 @@ export { ensureVirtualenv, pipInstall, pipUninstall, - pipInstallE, - pipenvInstall, - setWorkonHome, + setupPyInstall, }; const binDirName = process.platform === 'win32' ? 'Scripts' : 'bin'; -interface PipenvOptions { - dev?: boolean; -} - function getActiveVenvName() { return process.env.VIRTUAL_ENV ? path.basename(process.env.VIRTUAL_ENV) @@ -109,14 +102,14 @@ function createVenv(venvDir: string) { revert = deactivateVirtualenv(); } try { - let proc = subProcess.executeSync('virtualenv', [venvDir]); + let proc = subProcess.executeSync('python3 -m venv', [venvDir]); if (proc.status !== 0) { console.error(proc.stdout.toString() + '\n' + proc.stderr.toString()); throw new Error('Failed to create virtualenv in ' + venvDir); } if (process.env.PIP_VER) { proc = subProcess.executeSync( - path.resolve(venvDir, binDirName, 'python'), + path.resolve(venvDir, binDirName, 'python3'), ['-m', 'pip', 'install', `pip==${process.env.PIP_VER}`] ); if (proc.status !== 0) { @@ -148,16 +141,12 @@ function pipInstall() { } } -function pipInstallE() { - const proc = subProcess.executeSync('pip', [ - 'install', - '-e', - '.', - '--disable-pip-version-check', - ]); +function setupPyInstall() { + const proc = subProcess.executeSync('python3', ['setup.py', 'install']); if (proc.status !== 0) { + console.log('' + proc.stderr); throw new Error( - 'Failed to install requirements with pip.' + + 'Failed to install requirements with setup.py.' + ' venv = ' + JSON.stringify(getActiveVenvName()) ); @@ -177,41 +166,6 @@ function pipUninstall(pkgName: string) { } } -function pipenvInstall(options?: PipenvOptions) { - try { - const args = ['update']; - if (options?.dev) { - args.push('--dev'); - } - const proc = subProcess.executeSync('pipenv', [...args]); - if (proc.status !== 0) { - console.log('' + proc.stderr); - } - } finally { - try { - fs.unlinkSync('Pipfile.lock'); - } catch (e) { - // will error if the file doesn't exist, which is fine - } - } -} - -function setWorkonHome() { - const venvsBaseDir = path.join(path.resolve(__dirname), '.venvs'); - try { - fs.accessSync(venvsBaseDir, fs.constants.R_OK); - } catch (e) { - fs.mkdirSync(venvsBaseDir); - } - - const origWorkonHome = process.env.WORKON_HOME; - process.env.WORKON_HOME = venvsBaseDir; - - return function revert() { - process.env.WORKON_HOME = origWorkonHome; - }; -} - export function chdirWorkspaces(dir: string) { process.chdir(path.resolve(__dirname, 'workspaces', dir)); } diff --git a/test/tsconfig.json b/test/tsconfig.json deleted file mode 100644 index e516b7fa..00000000 --- a/test/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../tsconfig.json", - "include": ["**/*.ts"] -} diff --git a/test/unit/setup_file.spec.ts b/test/unit/setup_file.spec.ts index 9c8d71e0..552083dd 100644 --- a/test/unit/setup_file.spec.ts +++ b/test/unit/setup_file.spec.ts @@ -8,7 +8,7 @@ describe('Test setup_file.py', () => { ${'from setuptools import setup;setup(name="test")'} `("parse works for '$setupPyContent'", ({ setupPyContent }) => { const result = executeSync( - 'python', + 'python3', ['-c', `from setup_file import parse; parse('${setupPyContent}')`], { cwd: path.resolve(__dirname, '../../pysrc') } ); @@ -23,7 +23,7 @@ describe('Test setup_file.py', () => { ); const result = executeSync( - 'python', + 'python3', [ '-c', `from pip_resolve import get_requirements_list; get_requirements_list('${fixturePath}', True)`, diff --git a/test/workspaces/pip-app-with-options/requirements.txt b/test/workspaces/pip-app-with-options/requirements.txt deleted file mode 100644 index b9046ba2..00000000 --- a/test/workspaces/pip-app-with-options/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ ---require-hashes -MarkupSafe==1.1.1 --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \ - --hash=sha256:abcd \ - --hash=sha256:0123 # a comment -dnspython==1.13.0 \ - --hash=sha256:80f89881b402fc3b931a936111b43bcfe3abd8b0005d27e50e3c5fb59f7260f8 \ - --install-option="--no-compile" - diff --git a/test/workspaces/pip-app-with-repeating-dependency/requirements.txt b/test/workspaces/pip-app-with-repeating-dependency/requirements.txt deleted file mode 100644 index 5f5ddc5d..00000000 --- a/test/workspaces/pip-app-with-repeating-dependency/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# the gevent package has 'psutil' twice as a dependency (probably with different requirements) -gevent==21.8.0 diff --git a/test/workspaces/pip-app-without-markupsafe/requirements.txt b/test/workspaces/pip-app-without-markupsafe/requirements.txt new file mode 100644 index 00000000..c3eeef4f --- /dev/null +++ b/test/workspaces/pip-app-without-markupsafe/requirements.txt @@ -0,0 +1 @@ +Jinja2==2.7.2 diff --git a/test/workspaces/pip-app/requirements.txt b/test/workspaces/pip-app/requirements.txt index ee87e6f8..52c97948 100644 --- a/test/workspaces/pip-app/requirements.txt +++ b/test/workspaces/pip-app/requirements.txt @@ -1,8 +1,9 @@ Jinja2==2.7.2 Django==1.6.1 python-etcd==0.4.5 +urllib3==1.26.16 Django-Select2==6.0.1 # this version installs with lowercase so it catches a previous bug in pip_resolve.py -irc==16.2 # this has a cyclic dependecy (interanl jaraco.text <==> jaraco.collections) +irc==16.2 # this has a cyclic dependency (internal jaraco.text <==> jaraco.collections) testtools==\ - 2.3.0 # this has a cycle (fixtures ==> testtols); -./packages/prometheus_client-0.6.0 \ No newline at end of file + 2.3.0 # this has a cycle (fixtures ==> testtools); +./packages/prometheus_client-0.6.0 diff --git a/test/workspaces/pipenv-app/Pipfile b/test/workspaces/pipenv-app/Pipfile index 818c0425..8e400f6c 100644 --- a/test/workspaces/pipenv-app/Pipfile +++ b/test/workspaces/pipenv-app/Pipfile @@ -5,10 +5,11 @@ name = "pypi" [packages] python-etcd = ">=0.4,<0.5" +urllib3 = "1.26.16" testtools = "*" "Jinja2" = { version = "*" } [dev-packages] bs4 = "*" -[requires] \ No newline at end of file +[requires] diff --git a/test/workspaces/pipfile-nested-dirs/nested/directory/Pipfile b/test/workspaces/pipfile-nested-dirs/nested/directory/Pipfile index 19823db0..7fbcdd29 100644 --- a/test/workspaces/pipfile-nested-dirs/nested/directory/Pipfile +++ b/test/workspaces/pipfile-nested-dirs/nested/directory/Pipfile @@ -5,6 +5,7 @@ name = "pypi" [packages] python-etcd = ">=0.4,<0.5" +urllib3 = "1.26.16" testtools = "*" "Jinja2" = { version = "*" } django = { git = 'https://github.com/django/django.git', ref = '1.6.1', editable = true } diff --git a/test/workspaces/pipfile-pipapp-pinned/Pipfile b/test/workspaces/pipfile-pipapp-pinned/Pipfile index eb087de7..92a51296 100644 --- a/test/workspaces/pipfile-pipapp-pinned/Pipfile +++ b/test/workspaces/pipfile-pipapp-pinned/Pipfile @@ -5,6 +5,7 @@ name = "pypi" [packages] python-etcd = "==0.4.5" +urllib3 = "==1.26.16" irc = "==16.2" testtools = "==2.3.0" "Jinja2" = "==2.7.2" diff --git a/test/workspaces/pipfile-pipapp/Pipfile b/test/workspaces/pipfile-pipapp/Pipfile index 19823db0..7499dfd0 100644 --- a/test/workspaces/pipfile-pipapp/Pipfile +++ b/test/workspaces/pipfile-pipapp/Pipfile @@ -5,6 +5,7 @@ name = "pypi" [packages] python-etcd = ">=0.4,<0.5" +urllib3 = "==1.26.16" testtools = "*" "Jinja2" = { version = "*" } django = { git = 'https://github.com/django/django.git', ref = '1.6.1', editable = true } diff --git a/test/workspaces/pipfile-pipapp/README b/test/workspaces/pipfile-pipapp/README index 8f8f215f..3bf9e816 100644 --- a/test/workspaces/pipfile-pipapp/README +++ b/test/workspaces/pipfile-pipapp/README @@ -1,2 +1,2 @@ This is a small pipenv-based config with a variety of types of dependency -defintions, based on pip-app and the pipenv example files. +definitions, based on pip-app and the pipenv example files. diff --git a/test/workspaces/setup_py-app/setup.py b/test/workspaces/setup_py-app/setup.py index ff8a387b..6ac78084 100644 --- a/test/workspaces/setup_py-app/setup.py +++ b/test/workspaces/setup_py-app/setup.py @@ -5,12 +5,13 @@ setup( name="test_package", version="1.0.2", - packages=[ + install_requires=[ "Jinja2==2.7.2", "Django==1.6.1", "python-etcd==0.4.5", + "urllib3==1.26.16", "Django-Select2==6.0.1", # this version installs with lowercase so it catches a previous bug in pip_resolve.py - "irc==16.2", # this has a cyclic dependecy (interanl jaraco.text <==> jaraco.collections) - "testtools==2.3.0", # this has a cycle (fixtures ==> testtols) + "irc==16.2", # this has a cyclic dependency (internal jaraco.text <==> jaraco.collections) + "testtools==2.3.0", # this has a cycle (fixtures ==> testtools) ], )