Fix/pip attempt 2

.circleci/continue_config.yml

@@ -160,6 +160,9 @@ commands:
         # are best handled by conda-installing instead of pip-installing them.
         # Dependency resolution works best when installing these altogether in one
         # `conda install` command rather than one at a time in several sequential `conda install`s.
+      - run:
+          name: Install pip setuptools
+          command: conda activate kedro_builder; make install-pip-setuptools
       - run:
           name: Install GDAL, Fiona and pytables
           command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y
@@ -286,14 +289,14 @@ jobs:
           steps:
             - run:
                 name: Run unit tests without spark in parallel
-                command: conda activate kedro_builder; make test-no-spark
+                command: conda activate kedro_builder; pip -V; make test-no-spark
       - when:
           condition:
             equal: [ "3.10", <<parameters.python_version>> ]
           steps:
             - run:
                 name: Run unit tests without spark sequentially
-                command: conda activate kedro_builder; pytest tests --no-cov --ignore tests/extras/datasets/spark
+                command: conda activate kedro_builder; pip -V; pytest tests --no-cov --ignore tests/extras/datasets/spark
 
   lint:
     parameters:

Makefile

@@ -8,19 +8,19 @@ clean:
 	pre-commit clean || true
 
 install-pip-setuptools:
-	pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel
+	python -m pip install -U "pip==23.1.2" "setuptools>=65.5.1" wheel
 
 lint:
 	pre-commit run -a --hook-stage manual $(hook)
 
 test:
-	pytest --numprocesses 4 --dist loadfile
+	pytest tests/framework/cli/micropkg
 
 test-no-spark:
-	pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile
+	pytest tests/framework/cli/micropkg --no-cov --ignore tests/extras/datasets/spark --numprocesses logical --dist loadfile
 
 test-no-datasets:
-	pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile
+	pytest tests/framework/cli/micropkg --no-cov --ignore tests/extras/datasets/ --numprocesses logical --dist loadfile
 
 e2e-tests:
 	behave

RELEASE.md

@@ -16,6 +16,7 @@
 
 ## Bug fixes and other changes
 * Consolidated dependencies and optional dependencies in `pyproject.toml`.
+* Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813
 
 ## Documentation changes
 

features/environment.py

@@ -103,7 +103,7 @@ def _setup_minimal_env(context):
             "pip",
             "install",
             "-U",
-            "pip>=21.2",
+            "pip==23.1.2",  # pip==23.2 breaks pip-tools 7.0 and not support Python 3.7
             "setuptools>=65.5.1",
             "wheel",
         ],

setup.py

@@ -181,7 +181,7 @@ def _collect_requirements(requires):
     "pyspark>=2.2, <4.0",
     "pytest-cov~=3.0",
     "pytest-mock>=1.7.1, <2.0",
-    "pytest-xdist[psutil]~=2.2.1",
+    "pytest-xdist[psutil]",
     "pytest~=7.2",
     "redis~=4.1",
     "requests-mock~=1.6",

tests/framework/cli/micropkg/test_micropkg_package.py

@@ -27,6 +27,10 @@ def assert_sdist_contents_correct(
 
         with tarfile.open(sdist_file, "r") as tar:
             sdist_contents = set(tar.getnames())
+        print(tar.closed)
+        tar.close()
+        print("after")
+        print(tar.closed)
 
         expected_files = {
             f"{package_name}-{version}/{package_name}/__init__.py",

tools/circleci/requirements.txt

@@ -1,3 +1,3 @@
-pip>=21.2
+pip==23.1.2
 setuptools>=65.5.1
 twine~=3.0