Use installer to remove custom unzip and spread code (bazelbuild#715)

mattem · Jul 7, 2022 · aa0cdd6 · aa0cdd6
1 parent f48181f
commit aa0cdd6
Show file tree

Hide file tree

Showing 10 changed files with 55 additions and 272 deletions.
diff --git a/examples/pip_install/pip_install_test.py b/examples/pip_install/pip_install_test.py
@@ -37,12 +37,11 @@ def test_data(self):
         self.assertListEqual(
             env.split(" "),
             [
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md",
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE",
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS",
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md",
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1",
-                "external/pip/pypi__s3cmd/s3cmd-2.1.0.data/scripts/s3cmd",
+                "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/INSTALL.md",
+                "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/LICENSE",
+                "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/NEWS",
+                "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/README.md",
+                "external/pip/pypi__s3cmd/data/share/man/man1/s3cmd.1",
             ],
         )
 
@@ -52,12 +51,13 @@ def test_dist_info(self):
         self.assertListEqual(
             env.split(" "),
             [
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/DESCRIPTION.rst",
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/METADATA",
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/RECORD",
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/WHEEL",
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/metadata.json",
-                "external/pip/pypi__boto3/boto3-1.14.51.dist-info/top_level.txt",
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/DESCRIPTION.rst",
+                'external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER',
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/METADATA",
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/RECORD",
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/WHEEL",
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/metadata.json",
+                "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/top_level.txt",
             ],
         )
 

diff --git a/examples/pip_parse/pip_parse_test.py b/examples/pip_parse/pip_parse_test.py
@@ -35,12 +35,11 @@ def test_data(self):
         self.assertListEqual(
             env.split(" "),
             [
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md",
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE",
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS",
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md",
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1",
-                "external/pypi_s3cmd/s3cmd-2.1.0.data/scripts/s3cmd",
+                "external/pypi_s3cmd/data/share/doc/packages/s3cmd/INSTALL.md",
+                "external/pypi_s3cmd/data/share/doc/packages/s3cmd/LICENSE",
+                "external/pypi_s3cmd/data/share/doc/packages/s3cmd/NEWS",
+                "external/pypi_s3cmd/data/share/doc/packages/s3cmd/README.md",
+                "external/pypi_s3cmd/data/share/man/man1/s3cmd.1",
             ],
         )
 
@@ -50,11 +49,12 @@ def test_dist_info(self):
         self.assertListEqual(
             env.split(" "),
             [
-                "external/pypi_requests/requests-2.25.1.dist-info/LICENSE",
-                "external/pypi_requests/requests-2.25.1.dist-info/METADATA",
-                "external/pypi_requests/requests-2.25.1.dist-info/RECORD",
-                "external/pypi_requests/requests-2.25.1.dist-info/WHEEL",
-                "external/pypi_requests/requests-2.25.1.dist-info/top_level.txt",
+                'external/pypi_requests/site-packages/requests-2.25.1.dist-info/INSTALLER',
+                "external/pypi_requests/site-packages/requests-2.25.1.dist-info/LICENSE",
+                "external/pypi_requests/site-packages/requests-2.25.1.dist-info/METADATA",
+                "external/pypi_requests/site-packages/requests-2.25.1.dist-info/RECORD",
+                "external/pypi_requests/site-packages/requests-2.25.1.dist-info/WHEEL",
+                "external/pypi_requests/site-packages/requests-2.25.1.dist-info/top_level.txt",
             ],
         )
 

diff --git a/examples/pip_repository_annotations/WORKSPACE b/examples/pip_repository_annotations/WORKSPACE
@@ -42,7 +42,7 @@ write_file(
         copy_executables = {"@pip_repository_annotations_example//:data/copy_executable.py": "copied_content/executable.py"},
         copy_files = {"@pip_repository_annotations_example//:data/copy_file.txt": "copied_content/file.txt"},
         data = [":generated_file"],
-        data_exclude_glob = ["*.dist-info/WHEEL"],
+        data_exclude_glob = ["site-packages/*.dist-info/WHEEL"],
     ),
 }
 

diff --git a/examples/pip_repository_annotations/pip_repository_annotations_test.py b/examples/pip_repository_annotations/pip_repository_annotations_test.py
@@ -69,7 +69,7 @@ def test_data_exclude_glob(self):
 
         r = runfiles.Create()
         dist_info_dir = (
-            "pip_repository_annotations_example/external/{}/wheel-{}.dist-info".format(
+            "pip_repository_annotations_example/external/{}/site-packages/wheel-{}.dist-info".format(
                 self.wheel_pkg_dir(),
                 current_wheel_version,
             )

diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD
@@ -9,7 +9,6 @@ py_library(
         "arguments.py",
         "bazel.py",
         "namespace_pkgs.py",
-        "purelib.py",
         "requirements.py",
         "wheel.py",
     ],
@@ -135,17 +134,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "purelib_test",
-    size = "small",
-    srcs = [
-        "purelib_test.py",
-    ],
-    deps = [
-        ":lib",
-    ],
-)
-
 filegroup(
     name = "distribution",
     srcs = glob(

diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py
@@ -9,7 +9,6 @@
 from python.pip_install.extract_wheels.lib import (
     annotation,
     namespace_pkgs,
-    purelib,
     wheel,
 )
 
@@ -137,27 +136,18 @@ def generate_build_file_contents(
     there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`).
     """
 
-    dist_info_ignores = [
-        # RECORD is known to contain sha256 checksums of files which might include the checksums
-        # of generated files produced when wheels are installed. The file is ignored to avoid
-        # Bazel caching issues.
-        "**/*.dist-info/RECORD",
-    ]
-
     data_exclude = list(
         set(
             [
-                "*.whl",
-                "**/__pycache__/**",
                 "**/* *",
                 "**/*.py",
                 "**/*.pyc",
-                "BUILD.bazel",
-                "WORKSPACE",
-                f"{WHEEL_ENTRY_POINT_PREFIX}*.py",
+                # RECORD is known to contain sha256 checksums of files which might include the checksums
+                # of generated files produced when wheels are installed. The file is ignored to avoid
+                # Bazel caching issues.
+                "**/*.dist-info/RECORD",
             ]
             + data_exclude
-            + dist_info_ignores
         )
     )
 
@@ -172,12 +162,12 @@ def generate_build_file_contents(
 
         filegroup(
             name = "{dist_info_label}",
-            srcs = glob(["*.dist-info/**"], allow_empty = True),
+            srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True),
         )
 
         filegroup(
             name = "{data_label}",
-            srcs = glob(["*.data/**"], allow_empty = True),
+            srcs = glob(["data/**"], allow_empty = True),
         )
 
         filegroup(
@@ -188,11 +178,11 @@ def generate_build_file_contents(
 
         py_library(
             name = "{name}",
-            srcs = glob(["**/*.py"], exclude={srcs_exclude}, allow_empty = True),
-            data = {data} + glob(["**/*"], exclude={data_exclude}),
+            srcs = glob(["site-packages/**/*.py"], exclude={srcs_exclude}, allow_empty = True),
+            data = {data} + glob(["site-packages/**/*"], exclude={data_exclude}),
             # This makes this directory a top-level in the python import
             # search path for anything that depends on this.
-            imports = ["."],
+            imports = ["site-packages"],
             deps = [{dependencies}],
             tags = [{tags}],
         )
@@ -377,9 +367,6 @@ def extract_wheel(
         shutil.copy(whl.path, directory)
     whl.unzip(directory)
 
-    # Note: Order of operations matters here
-    purelib.spread_purelib_into_root(directory)
-
     if not enable_implicit_namespace_pkgs:
         setup_namespace_pkg_compatibility(directory)
 

diff --git a/python/pip_install/extract_wheels/lib/purelib.py b/python/pip_install/extract_wheels/lib/purelib.py
diff --git a/python/pip_install/extract_wheels/lib/purelib_test.py b/python/pip_install/extract_wheels/lib/purelib_test.py