diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..9f7abf3163 --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro + +on: + push: + branches: + - main + paths-ignore: + - "docs/**" + pull_request: + branches: + - main + paths-ignore: + - "docs/**" + +jobs: + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.10" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml new file mode 100644 index 0000000000..aa53fdd604 --- /dev/null +++ b/.github/workflows/docs-only-checks.yml @@ -0,0 +1,24 @@ +name: Run linter on Kedro Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + pull_request: + branches: + - main + paths: + - "docs/**" + +jobs: + lint-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..6c317c7c40 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,46 @@ +name: Run e2e-tests on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string + +env: + COLUMNS: 120 + LINES: 25 + +jobs: + e2e-tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - run: make install-pip-setuptools + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + pip --version + make install-test-requirements + make install-pre-commit + - name: pip freeze + run: pip freeze + - name: Run e2e tests + run: make e2e-tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..81712415fd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,32 @@ +name: Run linters on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string + +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + make install-test-requirements + make install-pre-commit + pip freeze + - name: Run linter + run: make lint diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml new file mode 100644 index 0000000000..4f5393a1e6 --- /dev/null +++ b/.github/workflows/merge-gatekeeper.yml @@ -0,0 +1,27 @@ +name: Merge Gatekeeper + +on: + pull_request: + branches: + - main + - develop + +jobs: + merge-gatekeeper: + runs-on: ubuntu-latest + # Restrict permissions of the GITHUB_TOKEN. + # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs + permissions: + checks: read + statuses: read + steps: + - name: Run Merge Gatekeeper + # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs: + # https://github.com/upsidr/merge-gatekeeper/tags + # https://github.com/upsidr/merge-gatekeeper/branches + uses: upsidr/merge-gatekeeper@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: 1800 + interval: 30 + ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular' diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..87ca8e0eed --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,53 @@ +name: Run unit-tests on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit-tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - run: make install-pip-setuptools + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + make install-test-requirements + make install-pre-commit + - name: Install pytables (only for windows) + if: inputs.os == 'windows-latest' + run: pip install tables + - name: pip freeze + run: pip freeze + - name: Run unit tests + if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10' + run: make test-sequential + - name: Run unit tests + if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10' + run: make test + - name: Run unit tests (Windows) + if: inputs.os == 'windows-latest' && inputs.python-version == '3.10' + run: make test-no-spark-sequential + - name: Run unit tests (Windows) + if: inputs.os == 'windows-latest' && inputs.python-version != '3.10' + run: make test-no-spark diff --git a/Makefile b/Makefile index 5a1e85b558..bea1c7af02 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clean: pre-commit clean || true install-pip-setuptools: - pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel + python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel lint: pre-commit run -a --hook-stage manual $(hook) @@ -19,6 +19,12 @@ test: test-no-spark: pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile +test-sequential: + pytest tests --cov-config pyproject.toml + +test-no-spark-sequential: + pytest tests --no-cov --ignore tests/extras/datasets/spark + test-no-datasets: pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile diff --git a/RELEASE.md b/RELEASE.md index 0c0c40fd5c..2fa06b789f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ ## Bug fixes and other changes * Consolidated dependencies and optional dependencies in `pyproject.toml`. +* Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 ## Documentation changes diff --git a/features/environment.py b/features/environment.py index 218ff097e4..671bee0178 100644 --- a/features/environment.py +++ b/features/environment.py @@ -103,7 +103,9 @@ def _setup_minimal_env(context): "pip", "install", "-U", - "pip>=21.2", + # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7 + "pip>=21.2,<23.2; python_version < '3.8'", + "pip>=21.2; python_version >= '3.8'", "setuptools>=65.5.1", "wheel", ], diff --git a/pyproject.toml b/pyproject.toml index ae5fc96167..50bf4a8483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "more_itertools~=9.0", "omegaconf~=2.3", "parse~=1.19.0", - "pip-tools~=6.5", + "pip-tools>=6.5,<8", "pluggy~=1.0", "PyYAML>=4.2, <7.0", "rich>=12.0, <14.0", diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index 14719c544e..cb5551a327 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -9,3 +9,5 @@ static/img/kedro_gitflow.svg .coverage.* .*\.log .*\.iml +tests/extras/datasets/tensorflow/test_tensorflow_model_dataset.py +docs/source/meta/images/kedro_gitflow.svg