TEST-modin-project#2008: Run tests-all on push or if execution files …

…changed

Signed-off-by: Vasily Litvinov <fam1ly.n4me@yandex.ru>

.github/workflows/ci-required.yml

@@ -5,7 +5,11 @@ concurrency:
  # on old commits.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+env:
+ MODIN_GITHUB_CI: true
+
 jobs:
+
  check-pr-title:
  runs-on: ubuntu-latest
  steps:
@@ -14,6 +18,7 @@ jobs:
  # NOTE: If you change the allowed prefixes here, update
  # the documentation about them in /docs/development/contributing.rst
  regexp: '^(?:FEAT|DOCS|FIX|REFACTOR|TEST|PERF)-#\d+:'
+
  build-docs:
  name: build docs
  runs-on: ubuntu-latest
@@ -29,5 +34,77 @@ jobs:
  cache-dependency-path: '**/requirements-doc.txt'
  - run: pip install -r docs/requirements-doc.txt
  - run: cd docs && sphinx-build -T -E -W -b html . build
-env:
- MODIN_GITHUB_CI: true
+
+ lint-pydocstyle:
+ name: lint (pydocstyle)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: ./.github/actions/python-only
+ # The `numpydoc` version here MUST match the versions in the dev requirements files.
+ - run: pip install pytest pytest-cov pydocstyle numpydoc==1.1.0 xgboost
+ - run: python -m pytest scripts/test
+ - run: pip install -e ".[all]"
+ - run: |
+ python scripts/doc_checker.py --add-ignore=D101,D102,D103,D105 --disable-numpydoc \
+ modin/pandas/dataframe.py modin/pandas/series.py \
+ modin/pandas/groupby.py \
+ modin/pandas/series_utils.py modin/pandas/general.py \
+ modin/pandas/plotting.py modin/pandas/utils.py \
+ modin/pandas/iterator.py modin/pandas/indexing.py \
+ - run: python scripts/doc_checker.py modin/core/dataframe
+ - run: python scripts/doc_checker.py modin/core/execution/dask
+ - run: |
+ python scripts/doc_checker.py \
+ modin/pandas/accessor.py modin/pandas/general.py \
+ modin/pandas/groupby.py modin/pandas/indexing.py \
+ modin/pandas/iterator.py modin/pandas/plotting.py \
+ modin/pandas/series_utils.py modin/pandas/utils.py \
+ modin/pandas/base.py \
+ modin/pandas/io.py \
+ asv_bench/benchmarks/utils \
+ asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
+ asv_bench/benchmarks/scalability/__init__.py \
+ modin/core/io \
+ modin/experimental/core/execution/ray/implementations/pandas_on_ray \
+ modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
+ modin/pandas/series.py \
+ modin/core/execution/python \
+ modin/pandas/dataframe.py \
+ modin/config/__init__.py \
+ modin/config/__main__.py \
+ modin/config/envvars.py \
+ modin/config/pubsub.py
+ - run: python scripts/doc_checker.py modin/distributed
+ - run: python scripts/doc_checker.py modin/utils.py
+ - run: python scripts/doc_checker.py modin/experimental/sklearn
+ - run: |
+ python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \
+ modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \
+ modin/experimental/xgboost/xgboost_ray.py
+ - run: python scripts/doc_checker.py modin/core/execution/ray
+ - run: |
+ python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \
+ modin/core/execution/dispatching/factories/dispatcher.py \
+ - run: python scripts/doc_checker.py scripts/doc_checker.py
+ - run: |
+ python scripts/doc_checker.py modin/experimental/pandas/io.py \
+ modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py
+ - run: python scripts/doc_checker.py modin/core/storage_formats/base
+ - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/pyarrow
+ - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
+ - run: |
+ python scripts/doc_checker.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/io \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_serializer.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/df_algebra.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py \
+ modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py \
+ - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/hdk
+ - run: python scripts/doc_checker.py modin/experimental/core/execution/native/implementations/hdk_on_native/interchange/dataframe_protocol
+ - run: python scripts/doc_checker.py modin/experimental/batch/pipeline.py
+ - run: python scripts/doc_checker.py modin/logging

.github/workflows/ci.yml

@@ -24,6 +24,7 @@ concurrency:
  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 env:
  MODIN_GITHUB_CI: true
+
 jobs:
  lint-black:
  name: lint (black)
@@ -45,81 +46,6 @@ jobs:
  - run: pip install -r requirements-dev.txt
  - run: mypy --config-file mypy.ini
 
- lint-pydocstyle:
- if: github.event_name == 'pull_request'
- name: lint (pydocstyle)
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: ./.github/actions/python-only
- # The `numpydoc` version here MUST match the versions in the dev requirements files.
- - run: pip install pytest pytest-cov pydocstyle numpydoc==1.1.0 xgboost
- - run: python -m pytest scripts/test
- - run: pip install -e ".[all]"
- - run: |
- python scripts/doc_checker.py --add-ignore=D101,D102,D103,D105 --disable-numpydoc \
- modin/pandas/dataframe.py modin/pandas/series.py \
- modin/pandas/groupby.py \
- modin/pandas/series_utils.py modin/pandas/general.py \
- modin/pandas/plotting.py modin/pandas/utils.py \
- modin/pandas/iterator.py modin/pandas/indexing.py \
- - run: python scripts/doc_checker.py modin/core/dataframe
- - run: python scripts/doc_checker.py modin/core/execution/dask
- - run: |
- python scripts/doc_checker.py \
- modin/pandas/accessor.py modin/pandas/general.py \
- modin/pandas/groupby.py modin/pandas/indexing.py \
- modin/pandas/iterator.py modin/pandas/plotting.py \
- modin/pandas/series_utils.py modin/pandas/utils.py \
- modin/pandas/base.py \
- modin/pandas/io.py \
- asv_bench/benchmarks/utils \
- asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
- asv_bench/benchmarks/scalability/__init__.py \
- modin/core/io \
- modin/experimental/core/execution/ray/implementations/pandas_on_ray \
- modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
- modin/pandas/series.py \
- modin/core/execution/python \
- modin/pandas/dataframe.py \
- modin/config/__init__.py \
- modin/config/__main__.py \
- modin/config/envvars.py \
- modin/config/pubsub.py
- - run: python scripts/doc_checker.py modin/distributed
- - run: python scripts/doc_checker.py modin/utils.py
- - run: python scripts/doc_checker.py modin/experimental/sklearn
- - run: |
- python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \
- modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \
- modin/experimental/xgboost/xgboost_ray.py
- - run: python scripts/doc_checker.py modin/core/execution/ray
- - run: |
- python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \
- modin/core/execution/dispatching/factories/dispatcher.py \
- - run: python scripts/doc_checker.py scripts/doc_checker.py
- - run: |
- python scripts/doc_checker.py modin/experimental/pandas/io.py \
- modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py
- - run: python scripts/doc_checker.py modin/core/storage_formats/base
- - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/pyarrow
- - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
- - run: |
- python scripts/doc_checker.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe \
- modin/experimental/core/execution/native/implementations/hdk_on_native/io \
- modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning \
- modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_serializer.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/df_algebra.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py \
- modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py \
- - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/hdk
- - run: python scripts/doc_checker.py modin/experimental/core/execution/native/implementations/hdk_on_native/interchange/dataframe_protocol
- - run: python scripts/doc_checker.py modin/experimental/batch/pipeline.py
- - run: python scripts/doc_checker.py modin/logging
-
  lint-flake8:
  name: lint (flake8)
  runs-on: ubuntu-latest
@@ -347,8 +273,42 @@ jobs:
  path: asv_bench/benchmarks.log
  if: failure()
 
+ execution-filter:
+ # see if execution backend-specific changes were made
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: read
+ outputs:
+ ray: ${{ steps.filter.outputs.ray }}
+ dask: ${{ steps.filter.outputs.dask }}
+ unidist: ${{ steps.filter.outputs.unidist }}
+ engines: ${{ steps.engines.outputs.engines }}
+ steps:
+ - uses: actions/checkout@v3
+ - uses: dorny/paths-filter@v2
+ id: filter
+ with:
+ filters: |
+ shared: &shared
+ - 'modin/core/execution/dispatching/**'
+ ray:
+ - *shared
+ - 'modin/core/execution/ray/**'
+ dask:
+ - *shared
+ - 'modin/core/execution/dask/**'
+ unidist:
+ - *shared
+ - 'modin/core/execution/unidist/**'
+ - uses: actions/setup-python@v4
+ - id: engines
+ run: |
+ python -c "import sys, json; print('engines=' + json.dumps(['python'] + (sys.argv[1] == 'true' and ['ray'] or []) + (sys.argv[2] == 'true' and ['dask'] or []) ))" \
+ "${{ steps.filter.outputs.ray }}" "${{ steps.filter.outputs.dask }}" >> $GITHUB_OUTPUT
+
  test-all-unidist:
- needs: [lint-flake8, lint-black, lint-mypy, test-api-and-no-engine]
+ needs: [lint-flake8, lint-black, lint-mypy, test-api-and-no-engine, execution-filter]
+ if: github.event_name == 'push' || needs.execution-filter.outputs.unidist == 'true'
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -412,14 +372,14 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-all:
- needs: [lint-flake8, lint-black, lint-mypy, test-api-and-no-engine]
+ needs: [lint-flake8, lint-black, lint-mypy, test-api-and-no-engine, execution-filter]
  strategy:
  matrix:
  os:
  - ubuntu
  - windows
  python-version: ["3.8"]
- engine: ["python", "ray", "dask"]
+ engine: ${{ fromJSON(needs.execution-filter.outputs.engines) }}
  test_task:
  - group_1
  - group_2