fix: Pip not resolving local packages

aws_lambda_builders/workflows/python_pip/packager.py

@@ -596,7 +596,10 @@ def main(self, args, env_vars=None, shim=None):
 class PipRunner(object):
     """Wrapper around pip calls used by chalice."""
 
-    _LINK_IS_DIR_PATTERN = "Processing (.+?)\n  Link is a directory, ignoring download_dir"
+    # Update regex pattern to correspond with the updated output from pip
+    # Specific commit:
+    # https://github.com/pypa/pip/commit/b28e2c4928cc62d90b738a4613886fb1e2ad6a81#diff-5225c8e359020adb25dfc8c7a505950fd649c6c5775789c6f6517f7913f94542L529
+    _LINK_IS_DIR_PATTERNS = ["Processing (.+?)\n"]
 
     def __init__(self, python_exe, pip, osutils=None):
         if osutils is None:
@@ -644,10 +647,16 @@ def download_all_dependencies(self, requirements_filename, directory):
                 package_name = match.group(1)
                 raise NoSuchPackageError(str(package_name))
             raise PackageDownloadError(error)
+
+        # Extract local packages from pip output.
+        # Iterate over possible pip outputs depending on pip version.
         stdout = out.decode()
-        matches = re.finditer(self._LINK_IS_DIR_PATTERN, stdout)
-        for match in matches:
-            wheel_package_path = str(match.group(1))
+        wheel_package_paths = set()
+        for pattern in self._LINK_IS_DIR_PATTERNS:
+            for match in re.finditer(pattern, stdout):
+                wheel_package_paths.add(str(match.group(1)))
+
+        for wheel_package_path in wheel_package_paths:
             # Looks odd we do not check on the error status of building the
             # wheel here. We can assume this is a valid package path since
             # we already passed the pip download stage. This stage would have

tests/integration/workflows/python_pip/test_python_pip.py

@@ -27,7 +27,13 @@ def setUp(self):
         self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt")
         self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt")
 
-        self.test_data_files = {"__init__.py", "main.py", "requirements-invalid.txt", "requirements-numpy.txt"}
+        self.test_data_files = {
+            "__init__.py",
+            "main.py",
+            "requirements-invalid.txt",
+            "requirements-numpy.txt",
+            "local-dependencies",
+        }
 
         self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None)
         self.runtime = "{language}{major}.{minor}".format(
@@ -78,6 +84,27 @@ def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
                 self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8"
             )
 
+    def test_must_resolve_local_dependency(self):
+        source_dir = os.path.join(self.source_dir, "local-dependencies")
+        manifest = os.path.join(source_dir, "requirements.txt")
+        path_to_package = os.path.join(self.source_dir, "local-dependencies")
+        # pip resolves dependencies in requirements files relative to the current working directory
+        # need to make sure the correct path is used in the requirements file locally and in CI
+        with open(manifest, "w") as f:
+            f.write(str(path_to_package))
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime)
+        expected_files = {
+            "local_package",
+            "local_package-0.0.0.dist-info",
+            "requests",
+            "requests-2.23.0.dist-info",
+            "setup.py",
+            "requirements.txt",
+        }
+        output_files = set(os.listdir(self.artifacts_dir))
+        for f in expected_files:
+            self.assertIn(f, output_files)
+
     def test_must_fail_to_resolve_dependencies(self):
 
         with self.assertRaises(WorkflowFailedError) as ctx:

tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg

@@ -0,0 +1,16 @@
+[metadata]
+name = local_package
+description = "Use src/ setup of a python project to demonstrate local package resolution"
+version = 0.0.0
+
+[options]
+zip_safe = False
+packages = find:
+package_dir =
+    =src
+install_requires =
+    requests==2.23.0
+include_package_data = True
+
+[options.packages.find]
+where = src

tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py

@@ -0,0 +1,2 @@
+def handler(event, ctx):
+    pass

tests/unit/workflows/python_pip/test_packager.py

@@ -253,7 +253,7 @@ def test_download_wheels_no_wheels(self, pip_factory):
 
     def test_does_find_local_directory(self, pip_factory):
         pip, runner = pip_factory()
-        pip.add_return((0, (b"Processing ../local-dir\n" b"  Link is a directory," b" ignoring download_dir"), b""))
+        pip.add_return((0, b"Processing ../local-dir\n", b""))
         runner.download_all_dependencies("requirements.txt", "directory")
         assert len(pip.calls) == 2
         assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir"]
@@ -265,20 +265,21 @@ def test_does_find_multiple_local_directories(self, pip_factory):
                 0,
                 (
                     b"Processing ../local-dir-1\n"
-                    b"  Link is a directory,"
-                    b" ignoring download_dir"
                     b"\nsome pip output...\n"
                     b"Processing ../local-dir-2\n"
                     b"  Link is a directory,"
                     b" ignoring download_dir"
+                    b"Processing ../local-dir-3\n"
                 ),
                 b"",
             )
         )
         runner.download_all_dependencies("requirements.txt", "directory")
-        assert len(pip.calls) == 3
-        assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"]
-        assert pip.calls[2].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"]
+        pip_calls = [call.args for call in pip.calls]
+        assert len(pip.calls) == 4
+        assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] in pip_calls
+        assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] in pip_calls
+        assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-3"] in pip_calls
 
     def test_raise_no_such_package_error(self, pip_factory):
         pip, runner = pip_factory()