diff --git a/.github/workflows/ray_nightly_test.yml b/.github/workflows/ray_nightly_test.yml index 95b4eb96..9d6b056b 100644 --- a/.github/workflows/ray_nightly_test.yml +++ b/.github/workflows/ray_nightly_test.yml @@ -29,17 +29,21 @@ jobs: build-and-test: strategy: + fail-fast: false matrix: os: [ ubuntu-latest ] - python-version: [3.9, 3.10.14] + python-version: ['3.10', '3.11'] spark-version: [3.3.2, 3.4.0, 3.5.0] + exclude: + - python-version: '3.11' + spark-version: 3.3.2 runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2.3.4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Set up JDK 17 @@ -58,13 +62,13 @@ jobs: sudo apt-get install -y mpich - name: Cache pip - Ubuntu if: matrix.os == 'ubuntu-latest' - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: Cache pip - MacOS if: matrix.os == 'macos-latest' - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/Library/Caches/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -73,8 +77,7 @@ jobs: PYTHON_VERSION: ${{ matrix.python-version }} run: | python -m pip install --upgrade pip - pip install wheel - pip install "numpy<1.24" "click<8.3.0" + pip install wheel "numpy<2" "pandas<2" SUBVERSION=$(python -c 'import sys; print(sys.version_info[1])') if [ "$(uname -s)" == "Linux" ] then @@ -83,21 +86,21 @@ jobs: pip install torch fi case $PYTHON_VERSION in - 3.9) - pip install "ray[train,default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl" - ;; - 3.10.14) + 3.10) pip install "ray[train,default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl" ;; + 3.11) + pip install "ray[train,default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp311-cp311-manylinux2014_x86_64.whl" + ;; esac - pip install pyarrow tqdm pytest tensorflow==2.13.1 tabulate grpcio-tools wget + pip install pyarrow tqdm pytest pytest-rerunfailures "tensorflow-cpu>=2.13.1,<2.16" tabulate grpcio-tools wget pip install "xgboost_ray[default]<=0.1.13" pip install torchmetrics HOROVOD_WITH_GLOO=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch,ray] - name: Cache Maven - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/.m2 key: ${{ matrix.os }}-m2-${{ hashFiles('core/pom.xml') }} @@ -110,13 +113,15 @@ jobs: pip install dist/raydp-*.whl - name: Lint run: | - pip install pylint==2.8.3 + pip install pylint==3.2.7 pylint --rcfile=python/pylintrc python/raydp pylint --rcfile=python/pylintrc examples/*.py - name: Test with pytest + env: + CUDA_VISIBLE_DEVICES: "" run: | ray start --head --num-cpus 6 - pytest python/raydp/tests/ -v + pytest python/raydp/tests/ -v --reruns 2 --reruns-delay 10 ray stop --force - name: Test Examples run: | diff --git a/.github/workflows/raydp.yml b/.github/workflows/raydp.yml index a24746b9..97e870c4 100644 --- a/.github/workflows/raydp.yml +++ b/.github/workflows/raydp.yml @@ -30,18 +30,24 @@ permissions: # added using https://github.com/step-security/secure-repo jobs: build-and-test: strategy: + fail-fast: false matrix: os: [ubuntu-latest] - python-version: [3.9, 3.10.14] + python-version: ['3.10', '3.11'] spark-version: [3.3.2, 3.4.0, 3.5.0] ray-version: [2.37.0, 2.40.0, 2.50.0] + exclude: + - python-version: '3.11' + spark-version: 3.3.2 + - python-version: '3.11' + ray-version: 2.50.0 runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2.3.4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Set up JDK 17 @@ -60,22 +66,20 @@ jobs: sudo apt-get install -y mpich - name: Cache pip - Ubuntu if: matrix.os == 'ubuntu-latest' - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: Cache pip - MacOS if: matrix.os == 'macos-latest' - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/Library/Caches/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: Install dependencies run: | python -m pip install --upgrade pip - pip install wheel - pip install "numpy<1.24" "click<8.3.0" - pip install "pydantic<2.0" + pip install wheel "numpy<2" "pandas<2" "click<8.3.0" SUBVERSION=$(python -c 'import sys; print(sys.version_info[1])') if [ "$(uname -s)" == "Linux" ] then @@ -83,12 +87,12 @@ jobs: else pip install torch fi - pip install pyarrow "ray[train,default]==${{ matrix.ray-version }}" tqdm pytest tensorflow==2.13.1 tabulate grpcio-tools wget + pip install pyarrow "ray[train,default]==${{ matrix.ray-version }}" tqdm pytest pytest-rerunfailures "tensorflow-cpu>=2.13.1,<2.16" tabulate grpcio-tools wget pip install "xgboost_ray[default]<=0.1.13" pip install "xgboost<=2.0.3" pip install torchmetrics - name: Cache Maven - uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2 + uses: actions/cache@v4 with: path: ~/.m2 key: ${{ matrix.os }}-m2-${{ hashFiles('core/pom.xml') }} @@ -101,13 +105,15 @@ jobs: pip install dist/raydp-*.whl - name: Lint run: | - pip install pylint==2.8.3 + pip install pylint==3.2.7 pylint --rcfile=python/pylintrc python/raydp pylint --rcfile=python/pylintrc examples/*.py - name: Test with pytest + env: + CUDA_VISIBLE_DEVICES: "" run: | ray start --head --num-cpus 6 - pytest python/raydp/tests/ -v + pytest python/raydp/tests/ -v --reruns 2 --reruns-delay 10 ray stop --force - name: Test Examples run: | diff --git a/python/pylintrc b/python/pylintrc index 48bc7e2e..70f95bf1 100644 --- a/python/pylintrc +++ b/python/pylintrc @@ -49,7 +49,7 @@ unsafe-load-any-extension=no # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code -extension-pkg-whitelist=netifaces +extension-pkg-allow-list=netifaces [MESSAGES CONTROL] @@ -74,65 +74,35 @@ confidence= # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=abstract-method, - apply-builtin, arguments-differ, attribute-defined-outside-init, - backtick, - basestring-builtin, broad-except, - buffer-builtin, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, + broad-exception-raised, + consider-using-dict-items, + consider-using-f-string, + consider-using-from-import, + consider-using-generator, dangerous-default-value, - delslice-method, duplicate-code, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, fixme, - getslice-method, global-statement, - hex-method, + global-variable-not-assigned, import-error, import-self, - import-star-module-level, - input-builtin, - intern-builtin, invalid-name, locally-disabled, logging-fstring-interpolation, - long-builtin, - long-suffix, - map-builtin-not-iterating, missing-docstring, + missing-final-newline, missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, no-else-return, no-member, no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, + not-an-iterable, + not-callable, + possibly-used-before-assignment, protected-access, - raising-string, - range-builtin-not-iterating, redefined-outer-name, - reduce-builtin, - reload-builtin, - round-builtin, - setslice-method, - standarderror-builtin, suppressed-message, too-few-public-methods, too-many-ancestors, @@ -143,18 +113,17 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, too-many-statements, - unichr-builtin, - unicode-builtin, - unpacking-in-except, + unreachable, + unspecified-encoding, unused-argument, unused-import, unused-variable, + use-dict-literal, useless-else-on-loop, useless-suppression, - using-cmp-argument, + ungrouped-imports, wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, + wrong-import-position, [REPORTS] @@ -164,12 +133,6 @@ disable=abstract-method, # mypackage.mymodule.MyReporterClass. output-format=text -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - # Tells whether to display a full report or only the messages reports=no @@ -206,72 +169,39 @@ bad-names=foo,bar,baz,toto,tutu,tata # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty function-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - # Regular expression matching correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - # Regular expression matching correct constant names const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ -# Naming hint for constant names -const-name-hint=(([A-a-zZ_][A-Za-z0-9_]*)|(__.*__))$ - # Regular expression matching correct attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - # Regular expression matching correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - # Regular expression matching correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ @@ -326,12 +256,6 @@ ignore-long-lines=(?x)( # else. single-line-if-stmt=yes -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - # Maximum number of lines in a module max-module-lines=1000 @@ -481,6 +405,5 @@ valid-metaclass-classmethod-first-arg=mcs # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException +overgeneral-exceptions=builtins.Exception, + builtins.BaseException