From ff7ec7d5a86379da64437b88236f3dff6bb70d68 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 13:46:46 +0000 Subject: [PATCH 01/16] Pin `pip<23.2` to unblock CI from breaking changes Signed-off-by: Nok --- Makefile | 2 +- RELEASE.md | 1 + features/environment.py | 2 +- tools/circleci/requirements.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5a1e85b558..71d2caa4e1 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clean: pre-commit clean || true install-pip-setuptools: - pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel + pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel lint: pre-commit run -a --hook-stage manual $(hook) diff --git a/RELEASE.md b/RELEASE.md index 0c0c40fd5c..32397c5056 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ ## Bug fixes and other changes * Consolidated dependencies and optional dependencies in `pyproject.toml`. +* Pin `pip<23.2` due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 ## Documentation changes diff --git a/features/environment.py b/features/environment.py index 218ff097e4..ef0bf6691f 100644 --- a/features/environment.py +++ b/features/environment.py @@ -103,7 +103,7 @@ def _setup_minimal_env(context): "pip", "install", "-U", - "pip>=21.2", + "pip>=21.2, <23.2", # pip==23.2 breaks pip-tools 7.0 and not support Python 3.7 "setuptools>=65.5.1", "wheel", ], diff --git a/tools/circleci/requirements.txt b/tools/circleci/requirements.txt index 224e670115..7feecc1df1 100644 --- a/tools/circleci/requirements.txt +++ b/tools/circleci/requirements.txt @@ -1,3 +1,3 @@ -pip>=21.2 +pip>=21.2, <23.2 setuptools>=65.5.1 twine~=3.0 From 7af1a9e19f508e6c1e7da4414b5652a945b2a7a3 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 14:20:17 +0000 Subject: [PATCH 02/16] Add missing CI step for window tests Signed-off-by: Nok --- .circleci/continue_config.yml | 3 +++ RELEASE.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index c83d6615cb..5798515d6a 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -160,6 +160,9 @@ commands: # are best handled by conda-installing instead of pip-installing them. # Dependency resolution works best when installing these altogether in one # `conda install` command rather than one at a time in several sequential `conda install`s. + - run: + name: Install pip setuptools + command: make install-pip-setuptools - run: name: Install GDAL, Fiona and pytables command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y diff --git a/RELEASE.md b/RELEASE.md index 32397c5056..2fa06b789f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,7 +16,7 @@ ## Bug fixes and other changes * Consolidated dependencies and optional dependencies in `pyproject.toml`. -* Pin `pip<23.2` due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 +* Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 ## Documentation changes From 4e9d50a38ed21cdb0694ce00c8f4b419e8864b96 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 14:32:08 +0000 Subject: [PATCH 03/16] fix CI config, activate conda env Signed-off-by: Nok --- .circleci/continue_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5798515d6a..c9653a0879 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -162,7 +162,7 @@ commands: # `conda install` command rather than one at a time in several sequential `conda install`s. - run: name: Install pip setuptools - command: make install-pip-setuptools + command: conda activate kedro_builder; make install-pip-setuptools - run: name: Install GDAL, Fiona and pytables command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y From 3cb9d28d068695ca6b8607d9b5e2660cffdc442c Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 14:47:53 +0000 Subject: [PATCH 04/16] update makefile to install pip correctly Signed-off-by: Nok --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71d2caa4e1..779d50e54a 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clean: pre-commit clean || true install-pip-setuptools: - pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel + python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel lint: pre-commit run -a --hook-stage manual $(hook) From 74a4f7accf8b4ad05fb4c1397cf01a4cb4a49853 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 23:16:30 +0000 Subject: [PATCH 05/16] temporarily remove multi-process of pytest for debugging Signed-off-by: Nok --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 779d50e54a..c7f525ffd8 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ lint: pre-commit run -a --hook-stage manual $(hook) test: - pytest --numprocesses 4 --dist loadfile + pytest test-no-spark: pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile From a10606aee190a78be4d5d3e1c9fbb21e14587cc1 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 23:20:54 +0000 Subject: [PATCH 06/16] change config temporarily for debuging Signed-off-by: Nok --- .circleci/continue_config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index c9653a0879..2016fe0864 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -287,6 +287,9 @@ jobs: condition: equal: [ "3.10", <> ] steps: + - run: + name: Install pip setuptools + command: conda activate kedro_builder; make install-pip-setuptools - run: name: Run unit tests without spark in parallel command: conda activate kedro_builder; make test-no-spark From 04febbdc4a8b84cd9e379a9488e44903ccedc083 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 18 Jul 2023 23:50:46 +0000 Subject: [PATCH 07/16] Speed up debugging, need to revert Signed-off-by: Nok --- .circleci/continue_config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 2016fe0864..b5cd5e36dd 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -287,9 +287,6 @@ jobs: condition: equal: [ "3.10", <> ] steps: - - run: - name: Install pip setuptools - command: conda activate kedro_builder; make install-pip-setuptools - run: name: Run unit tests without spark in parallel command: conda activate kedro_builder; make test-no-spark @@ -299,7 +296,7 @@ jobs: steps: - run: name: Run unit tests without spark sequentially - command: conda activate kedro_builder; pytest tests --no-cov --ignore tests/extras/datasets/spark + command: conda activate kedro_builder; pip -V; pytest tests/framework/cli/micropkg/test_micropkg_package.py lint: parameters: From c20ba7f3f924ef830d04eb2d2e8e03932155d322 Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 20 Jul 2023 22:47:14 +0000 Subject: [PATCH 08/16] Bump pytest-xdist, no error when run in non-parallel Signed-off-by: Nok --- .circleci/continue_config.yml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index b5cd5e36dd..4596294707 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -289,14 +289,14 @@ jobs: steps: - run: name: Run unit tests without spark in parallel - command: conda activate kedro_builder; make test-no-spark + command: conda activate kedro_builder; pip -V; make test-no-spark - when: condition: equal: [ "3.10", <> ] steps: - run: name: Run unit tests without spark sequentially - command: conda activate kedro_builder; pip -V; pytest tests/framework/cli/micropkg/test_micropkg_package.py + command: conda activate kedro_builder; pip -V; pytest tests --no-cov --ignore tests/extras/datasets/spark lint: parameters: diff --git a/setup.py b/setup.py index b7ba8b0988..e747691387 100644 --- a/setup.py +++ b/setup.py @@ -181,7 +181,7 @@ def _collect_requirements(requires): "pyspark>=2.2, <4.0", "pytest-cov~=3.0", "pytest-mock>=1.7.1, <2.0", - "pytest-xdist[psutil]~=2.2.1", + "pytest-xdist[psutil]", "pytest~=7.2", "redis~=4.1", "requests-mock~=1.6", From cfd16d092856952bda519b43ec8ebe94a49381ff Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 20 Jul 2023 23:44:51 +0000 Subject: [PATCH 09/16] use --numprocess logical instead of hard code processes Signed-off-by: Nok --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c7f525ffd8..b01c29a238 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,10 @@ test: pytest test-no-spark: - pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile + pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses logical --dist loadfile test-no-datasets: - pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile + pytest --no-cov --ignore tests/extras/datasets/ --numprocesses logical --dist loadfile e2e-tests: behave From 80d2894f6580208a330c806fc7359580d63a876e Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 20 Jul 2023 23:46:58 +0000 Subject: [PATCH 10/16] [REVERT LATER] - temp replace 3.10 = 3.7 Signed-off-by: Nok --- .circleci/continue_config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 4596294707..1e703fa204 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -151,7 +151,7 @@ commands: # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll). - unless: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - restore_cache: name: Restore conda environment cache @@ -247,14 +247,14 @@ jobs: - setup - unless: condition: - equal: ["3.10", <>] + equal: ["3.7", <>] steps: - run: name: Run unit tests in parallel command: PYTEST_ADDOPTS="-v" make test - when: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - run: name: Run unit tests sequentially @@ -285,14 +285,14 @@ jobs: command: setx /m HDF5_DISABLE_VERSION_CHECK 1 - unless: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - run: name: Run unit tests without spark in parallel command: conda activate kedro_builder; pip -V; make test-no-spark - when: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - run: name: Run unit tests without spark sequentially @@ -349,7 +349,7 @@ jobs: # 'Install GDAL, Fiona and pytables' and 'Restore conda environment cache' steps. - unless: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - save_cache: name: Save conda environment cache @@ -445,14 +445,14 @@ jobs: command: make lint - unless: condition: - equal: ["3.10", <>] + equal: [3.7", <>] steps: - run: name: Run unit tests in parallel command: make test - when: condition: - equal: [ "3.10", <> ] + equal: [ "3.7", <> ] steps: - run: name: Run unit tests sequentially From 556750742a2b098f4a3ab3bbc0b04371debab848 Mon Sep 17 00:00:00 2001 From: Nok Date: Fri, 21 Jul 2023 16:08:43 +0000 Subject: [PATCH 11/16] Revert "[REVERT LATER] - temp replace 3.10 = 3.7" This reverts commit 80d2894f6580208a330c806fc7359580d63a876e. --- .circleci/continue_config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 1e703fa204..4596294707 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -151,7 +151,7 @@ commands: # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll). - unless: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - restore_cache: name: Restore conda environment cache @@ -247,14 +247,14 @@ jobs: - setup - unless: condition: - equal: ["3.7", <>] + equal: ["3.10", <>] steps: - run: name: Run unit tests in parallel command: PYTEST_ADDOPTS="-v" make test - when: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - run: name: Run unit tests sequentially @@ -285,14 +285,14 @@ jobs: command: setx /m HDF5_DISABLE_VERSION_CHECK 1 - unless: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - run: name: Run unit tests without spark in parallel command: conda activate kedro_builder; pip -V; make test-no-spark - when: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - run: name: Run unit tests without spark sequentially @@ -349,7 +349,7 @@ jobs: # 'Install GDAL, Fiona and pytables' and 'Restore conda environment cache' steps. - unless: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - save_cache: name: Save conda environment cache @@ -445,14 +445,14 @@ jobs: command: make lint - unless: condition: - equal: [3.7", <>] + equal: ["3.10", <>] steps: - run: name: Run unit tests in parallel command: make test - when: condition: - equal: [ "3.7", <> ] + equal: [ "3.10", <> ] steps: - run: name: Run unit tests sequentially From 6a76e2d1f45709b94409e985f4ea88001df7acf0 Mon Sep 17 00:00:00 2001 From: Nok Date: Fri, 21 Jul 2023 16:30:11 +0000 Subject: [PATCH 12/16] Only execute micropkg tests to speed up debuging Signed-off-by: Nok --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b01c29a238..a498d16a15 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,13 @@ lint: pre-commit run -a --hook-stage manual $(hook) test: - pytest + pytest tests/framework/cli/micropkg test-no-spark: - pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses logical --dist loadfile + pytest tests/framework/cli/micropkg --no-cov --ignore tests/extras/datasets/spark --numprocesses logical --dist loadfile test-no-datasets: - pytest --no-cov --ignore tests/extras/datasets/ --numprocesses logical --dist loadfile + pytest tests/framework/cli/micropkg --no-cov --ignore tests/extras/datasets/ --numprocesses logical --dist loadfile e2e-tests: behave From aaa1007fdf12c00e19c4c3b89481b8680e37ce0a Mon Sep 17 00:00:00 2001 From: Nok Date: Fri, 21 Jul 2023 17:48:24 +0000 Subject: [PATCH 13/16] force pip version == 23.1.2 Signed-off-by: Nok --- Makefile | 2 +- features/environment.py | 2 +- tools/circleci/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a498d16a15..ea760dfeee 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clean: pre-commit clean || true install-pip-setuptools: - python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel + python -m pip install -U "pip==23.1.2" "setuptools>=65.5.1" wheel lint: pre-commit run -a --hook-stage manual $(hook) diff --git a/features/environment.py b/features/environment.py index ef0bf6691f..9ee6ba9de6 100644 --- a/features/environment.py +++ b/features/environment.py @@ -103,7 +103,7 @@ def _setup_minimal_env(context): "pip", "install", "-U", - "pip>=21.2, <23.2", # pip==23.2 breaks pip-tools 7.0 and not support Python 3.7 + "pip==23.1.2", # pip==23.2 breaks pip-tools 7.0 and not support Python 3.7 "setuptools>=65.5.1", "wheel", ], diff --git a/tools/circleci/requirements.txt b/tools/circleci/requirements.txt index 7feecc1df1..9a1192782e 100644 --- a/tools/circleci/requirements.txt +++ b/tools/circleci/requirements.txt @@ -1,3 +1,3 @@ -pip>=21.2, <23.2 +pip==23.1.2 setuptools>=65.5.1 twine~=3.0 From 7b32a4511e19dd993ba1edeece122150038c2085 Mon Sep 17 00:00:00 2001 From: Nok Date: Mon, 24 Jul 2023 10:31:22 +0000 Subject: [PATCH 14/16] Move pip setup to later because conda install GDAL updated pip Signed-off-by: Nok --- .circleci/continue_config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 4596294707..5246100559 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -160,12 +160,12 @@ commands: # are best handled by conda-installing instead of pip-installing them. # Dependency resolution works best when installing these altogether in one # `conda install` command rather than one at a time in several sequential `conda install`s. - - run: - name: Install pip setuptools - command: conda activate kedro_builder; make install-pip-setuptools - run: name: Install GDAL, Fiona and pytables command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y + - run: + name: Install pip setuptools + command: conda activate kedro_builder; make install-pip-setuptools - run: name: Show pip information command: conda activate kedro_builder; pip debug --verbose From 4ef3e7b86d72e5296ce1e416b7188238c019e05b Mon Sep 17 00:00:00 2001 From: Nok Date: Mon, 24 Jul 2023 11:17:07 +0000 Subject: [PATCH 15/16] Revert "Move pip setup to later because conda install GDAL updated pip" This reverts commit 7b32a4511e19dd993ba1edeece122150038c2085. --- .circleci/continue_config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5246100559..4596294707 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -160,12 +160,12 @@ commands: # are best handled by conda-installing instead of pip-installing them. # Dependency resolution works best when installing these altogether in one # `conda install` command rather than one at a time in several sequential `conda install`s. - - run: - name: Install GDAL, Fiona and pytables - command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y - run: name: Install pip setuptools command: conda activate kedro_builder; make install-pip-setuptools + - run: + name: Install GDAL, Fiona and pytables + command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y - run: name: Show pip information command: conda activate kedro_builder; pip debug --verbose From 1efda4ccba0d8d2cb6a2b7c208c0edafd64f771c Mon Sep 17 00:00:00 2001 From: Nok Date: Mon, 24 Jul 2023 11:21:28 +0000 Subject: [PATCH 16/16] force file handler to be closed Signed-off-by: Nok --- tests/framework/cli/micropkg/test_micropkg_package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/framework/cli/micropkg/test_micropkg_package.py b/tests/framework/cli/micropkg/test_micropkg_package.py index 4c3daf7abe..4c3d84458e 100644 --- a/tests/framework/cli/micropkg/test_micropkg_package.py +++ b/tests/framework/cli/micropkg/test_micropkg_package.py @@ -27,6 +27,10 @@ def assert_sdist_contents_correct( with tarfile.open(sdist_file, "r") as tar: sdist_contents = set(tar.getnames()) + print(tar.closed) + tar.close() + print("after") + print(tar.closed) expected_files = { f"{package_name}-{version}/{package_name}/__init__.py",