Skip to content

Commit

Permalink
Merge branch 'main' into feat/add-factories-cli-commands
Browse files Browse the repository at this point in the history
  • Loading branch information
AhdraMeraliQB authored Jul 27, 2023
2 parents 219729b + acff3e2 commit c346c17
Show file tree
Hide file tree
Showing 101 changed files with 567 additions and 382 deletions.
44 changes: 44 additions & 0 deletions .github/workflows/all-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: Run all checks on Kedro

on:
push:
branches:
- main
paths-ignore:
- "docs/**"
pull_request:
branches:
- main
paths-ignore:
- "docs/**"

jobs:
unit-tests:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/unit-tests.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}

lint:
strategy:
matrix:
os: [ ubuntu-latest ]
python-version: [ "3.10" ]
uses: ./.github/workflows/lint.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}

e2e-tests:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/e2e-tests.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
24 changes: 24 additions & 0 deletions .github/workflows/docs-only-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Run linter on Kedro Docs

on:
push:
branches:
- main
paths:
- "docs/**"
pull_request:
branches:
- main
paths:
- "docs/**"

jobs:
lint-tests:
strategy:
matrix:
os: [ ubuntu-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/lint.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
46 changes: 46 additions & 0 deletions .github/workflows/e2e-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Run e2e-tests on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string

env:
COLUMNS: 120
LINES: 25

jobs:
e2e-tests:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{inputs.python-version}}
uses: actions/setup-python@v3
with:
python-version: ${{inputs.python-version}}
- run: make install-pip-setuptools
- name: Cache python packages for Linux
if: inputs.os == 'ubuntu-latest'
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Cache python packages for Windows
if: inputs.os == 'windows-latest'
uses: actions/cache@v3
with:
path: ~\AppData\Local\pip\Cache
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
pip --version
make install-test-requirements
make install-pre-commit
- name: pip freeze
run: pip freeze
- name: Run e2e tests
run: make e2e-tests
32 changes: 32 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Run linters on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string

jobs:
lint:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ inputs.python-version }}
- name: Cache python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
make install-test-requirements
make install-pre-commit
pip freeze
- name: Run linter
run: make lint
27 changes: 27 additions & 0 deletions .github/workflows/merge-gatekeeper.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: Merge Gatekeeper

on:
pull_request:
branches:
- main
- develop

jobs:
merge-gatekeeper:
runs-on: ubuntu-latest
# Restrict permissions of the GITHUB_TOKEN.
# Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
permissions:
checks: read
statuses: read
steps:
- name: Run Merge Gatekeeper
# NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
# https://github.com/upsidr/merge-gatekeeper/tags
# https://github.com/upsidr/merge-gatekeeper/branches
uses: upsidr/merge-gatekeeper@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
timeout: 1800
interval: 30
ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular'
53 changes: 53 additions & 0 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Run unit-tests on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string
jobs:
unit-tests:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{inputs.python-version}}
uses: actions/setup-python@v3
with:
python-version: ${{inputs.python-version}}
- run: make install-pip-setuptools
- name: Cache python packages for Linux
if: inputs.os == 'ubuntu-latest'
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Cache python packages for Windows
if: inputs.os == 'windows-latest'
uses: actions/cache@v3
with:
path: ~\AppData\Local\pip\Cache
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
make install-test-requirements
make install-pre-commit
- name: Install pytables (only for windows)
if: inputs.os == 'windows-latest'
run: pip install tables
- name: pip freeze
run: pip freeze
- name: Run unit tests
if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10'
run: make test-sequential
- name: Run unit tests
if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10'
run: make test
- name: Run unit tests (Windows)
if: inputs.os == 'windows-latest' && inputs.python-version == '3.10'
run: make test-no-spark-sequential
- name: Run unit tests (Windows)
if: inputs.os == 'windows-latest' && inputs.python-version != '3.10'
run: make test-no-spark
111 changes: 42 additions & 69 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,57 @@ repos:
exclude: "^kedro/templates/|^features/steps/test_starter/"
- id: requirements-txt-fixer # Sorts entries in requirements.txt
exclude: "^kedro/templates/|^features/steps/test_starter/"

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.961
hooks:
- id: mypy
args: [--allow-redefinition, --ignore-missing-imports]
exclude: |
(?x)(
^kedro/templates/|
^docs/|
^features/steps/test_starter/
)
additional_dependencies:
- types-cachetools
- types-filelock
- types-PyYAML
- types-redis
- types-requests
- types-setuptools
- types-toml
- attrs

- repo: https://github.com/asottile/blacken-docs
rev: v1.12.1
hooks:
- id: blacken-docs
additional_dependencies: [black~=22.0]
entry: blacken-docs --skip-errors

- repo: https://github.com/asottile/pyupgrade
rev: v2.26.0
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.0.277
hooks:
- id: ruff
name: "ruff on kedro/"
args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix"]
exclude: "^kedro/templates/|^features/steps/test_starter/|tests|docs"
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.0.277
hooks:
- id: pyupgrade
args: [--py36-plus]
- id: ruff
name: "ruff on tests/ and docs/"
# PLR2004: Magic value used
# PLR0913: Too many arguments
args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix", "--ignore=PLR2004,PLR0913"]
# include: "tests"
exclude: "^kedro/templates/|^features/steps/test_starter/|kedro"

- repo: local
hooks:
- id: isort
name: "Sort imports"
language: system
types: [file, python]
exclude: ^kedro/templates/|^features/steps/test_starter
entry: isort
- id: black
name: "Black"
language: system
Expand All @@ -53,65 +83,6 @@ repos:
pass_filenames: false
entry: lint-imports


# It's impossible to specify per-directory configuration, so we just run it many times.
# https://github.com/PyCQA/pylint/issues/618
# The first set of pylint checks if for local pre-commit, it only runs on the files changed.
- id: pylint-quick-kedro
name: "Quick Pylint on kedro/*"
language: system
types: [file, python]
files: ^kedro/
exclude: ^kedro/templates/
entry: pylint -j 4 --disable=unnecessary-pass
stages: [commit]
- id: pylint-quick-tests
name: "Quick Pylint on tests/*"
language: system
types: [file, python]
files: ^tests/
entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods
stages: [commit]
- id: pylint-quick-features
name: "Quick Pylint on features/*"
language: system
types: [file, python]
files: ^features/
exclude: ^features/steps/test_starter
entry: pylint -j 4 --disable=missing-docstring,no-name-in-module
stages: [commit]

# The same pylint checks, but running on all files. It's for manual run with `make lint`
- id: pylint-kedro
name: "Pylint on kedro/*"
language: system
pass_filenames: false
stages: [manual]
entry: pylint -j 4 --disable=unnecessary-pass --init-hook="import sys; sys.setrecursionlimit(2000)" kedro
- id: pylint-tests
name: "Pylint on tests/*"
language: system
pass_filenames: false
stages: [manual]
entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods,use-implicit-booleaness-not-comparison tests
- id: pylint-features
name: "Pylint on features/*"
language: system
pass_filenames: false
stages: [manual]
exclude: ^features/steps/test_starter
entry: pylint -j 4 --disable=missing-docstring,no-name-in-module features
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.2.3
hooks:
- id: flake8
args:
- "--max-line-length=88"
- "--max-complexity=18"
- "--max-complexity=18"
- "--select=B,C,E,F,W,T4,B9"
- "--ignore=E203,E266,E501,W503"
exclude: "^kedro/templates/|^features/steps/test_starter/"
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.961
hooks:
Expand Down Expand Up @@ -147,3 +118,5 @@ repos:
types: [file, python]
exclude: ^kedro/templates/|^tests/|^features/steps/test_starter
entry: bandit -ll

# Manual only
9 changes: 7 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,22 @@ clean:
pre-commit clean || true

install-pip-setuptools:
pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel
python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel

lint:
pre-commit run -a --hook-stage manual $(hook)

test:
pytest --numprocesses 4 --dist loadfile

test-no-spark:
pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile

test-sequential:
pytest tests --cov-config pyproject.toml

test-no-spark-sequential:
pytest tests --no-cov --ignore tests/extras/datasets/spark

test-no-datasets:
pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile

Expand Down
Loading

0 comments on commit c346c17

Please sign in to comment.