diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 34058602b..8cebd15ee 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -596,7 +596,10 @@ def main(self, args, env_vars=None, shim=None): class PipRunner(object): """Wrapper around pip calls used by chalice.""" - _LINK_IS_DIR_PATTERN = "Processing (.+?)\n Link is a directory, ignoring download_dir" + # Update regex pattern to correspond with the updated output from pip + # Specific commit: + # https://github.com/pypa/pip/commit/b28e2c4928cc62d90b738a4613886fb1e2ad6a81#diff-5225c8e359020adb25dfc8c7a505950fd649c6c5775789c6f6517f7913f94542L529 + _LINK_IS_DIR_PATTERNS = ["Processing (.+?)\n"] def __init__(self, python_exe, pip, osutils=None): if osutils is None: @@ -644,10 +647,16 @@ def download_all_dependencies(self, requirements_filename, directory): package_name = match.group(1) raise NoSuchPackageError(str(package_name)) raise PackageDownloadError(error) + + # Extract local packages from pip output. + # Iterate over possible pip outputs depending on pip version. stdout = out.decode() - matches = re.finditer(self._LINK_IS_DIR_PATTERN, stdout) - for match in matches: - wheel_package_path = str(match.group(1)) + wheel_package_paths = set() + for pattern in self._LINK_IS_DIR_PATTERNS: + for match in re.finditer(pattern, stdout): + wheel_package_paths.add(str(match.group(1))) + + for wheel_package_path in wheel_package_paths: # Looks odd we do not check on the error status of building the # wheel here. We can assume this is a valid package path since # we already passed the pip download stage. This stage would have diff --git a/tests/integration/workflows/__init__.py b/tests/integration/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/__init__.py b/tests/integration/workflows/python_pip/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 265b4513f..19529779a 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -27,7 +27,13 @@ def setUp(self): self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt") self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt") - self.test_data_files = {"__init__.py", "main.py", "requirements-invalid.txt", "requirements-numpy.txt"} + self.test_data_files = { + "__init__.py", + "main.py", + "requirements-invalid.txt", + "requirements-numpy.txt", + "local-dependencies", + } self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None) self.runtime = "{language}{major}.{minor}".format( @@ -78,6 +84,27 @@ def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8" ) + def test_must_resolve_local_dependency(self): + source_dir = os.path.join(self.source_dir, "local-dependencies") + manifest = os.path.join(source_dir, "requirements.txt") + path_to_package = os.path.join(self.source_dir, "local-dependencies") + # pip resolves dependencies in requirements files relative to the current working directory + # need to make sure the correct path is used in the requirements file locally and in CI + with open(manifest, "w") as f: + f.write(str(path_to_package)) + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime) + expected_files = { + "local_package", + "local_package-0.0.0.dist-info", + "requests", + "requests-2.23.0.dist-info", + "setup.py", + "requirements.txt", + } + output_files = set(os.listdir(self.artifacts_dir)) + for f in expected_files: + self.assertIn(f, output_files) + def test_must_fail_to_resolve_dependencies(self): with self.assertRaises(WorkflowFailedError) as ctx: diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg new file mode 100644 index 000000000..86534d155 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg @@ -0,0 +1,16 @@ +[metadata] +name = local_package +description = "Use src/ setup of a python project to demonstrate local package resolution" +version = 0.0.0 + +[options] +zip_safe = False +packages = find: +package_dir = + =src +install_requires = + requests==2.23.0 +include_package_data = True + +[options.packages.find] +where = src diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py new file mode 100644 index 000000000..606849326 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/__init__.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py new file mode 100644 index 000000000..e57ab293d --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py @@ -0,0 +1,2 @@ +def handler(event, ctx): + pass diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index af3a4c270..236b4655f 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -253,7 +253,7 @@ def test_download_wheels_no_wheels(self, pip_factory): def test_does_find_local_directory(self, pip_factory): pip, runner = pip_factory() - pip.add_return((0, (b"Processing ../local-dir\n" b" Link is a directory," b" ignoring download_dir"), b"")) + pip.add_return((0, b"Processing ../local-dir\n", b"")) runner.download_all_dependencies("requirements.txt", "directory") assert len(pip.calls) == 2 assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir"] @@ -265,20 +265,21 @@ def test_does_find_multiple_local_directories(self, pip_factory): 0, ( b"Processing ../local-dir-1\n" - b" Link is a directory," - b" ignoring download_dir" b"\nsome pip output...\n" b"Processing ../local-dir-2\n" b" Link is a directory," b" ignoring download_dir" + b"Processing ../local-dir-3\n" ), b"", ) ) runner.download_all_dependencies("requirements.txt", "directory") - assert len(pip.calls) == 3 - assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] - assert pip.calls[2].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] + pip_calls = [call.args for call in pip.calls] + assert len(pip.calls) == 4 + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] in pip_calls + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] in pip_calls + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-3"] in pip_calls def test_raise_no_such_package_error(self, pip_factory): pip, runner = pip_factory()