diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3162f1839b98..3ed9e6dc29b61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,7 +66,7 @@ repos: - id: flake8 args: ["--config=.flake8"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.272 + rev: v0.0.287 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --no-cache] diff --git a/pyproject.toml b/pyproject.toml index 86578b38dec71..3737bfff7ae36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,11 +40,14 @@ select = [ # NumPy-specific rules "NPY001", + # "NPY003", # Bugbear "B002", "B003", "B004", + # "B006", # To be confirmed + # "B007", "B009", "B010", "B011", @@ -69,6 +72,10 @@ select = [ "PLC3002", "PLR0206", "PLR0402", + # "PLR1701", + # "PLR1711", + # "PLR1722", + # "PLW3301", ] unfixable = [ "NPY001" @@ -82,6 +89,25 @@ ignore = [ "UP015", # It will cause the performance regression on python3.10 "UP038", + + "F403", # Open Later + "F522", # Open Later + "F811", # Open Later + "F821", # Open Later + "C408", # Open Later + "C403", # Open Later + "C405", # Open Later + "C417", # Open Later + "C411", # Open Later + "C416", # Open Later + "UP032", # Open Later + "UP015", # Open Later + "UP018", # Open Later + "UP031", # Open Later + "UP030", # Open Later + "UP028", # Open Later + "UP034", # Open Later + "UP004", # Open Later ] [tool.ruff.per-file-ignores] diff --git a/python/paddle/base/compiler.py b/python/paddle/base/compiler.py index b1944ac343b8b..381beba1f26ee 100644 --- a/python/paddle/base/compiler.py +++ b/python/paddle/base/compiler.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing -import os import sys import warnings from . import framework -from .framework import _get_paddle_place, _get_paddle_place_list +from .framework import _get_paddle_place, _get_paddle_place_list # noqa: F401 from .framework import cuda_places, cpu_places, xpu_places from . import core diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index 720c3835fe1cf..78275af98cc65 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -17,7 +17,6 @@ import os import warnings import platform -import logging has_paddle_dy_lib = False @@ -280,34 +279,45 @@ def to_list(s): libpaddle.LoDTensor = libpaddle.Tensor from .libpaddle import * - from .libpaddle import __doc__, __file__, __name__, __package__ - from .libpaddle import __unittest_throw_exception__ - from .libpaddle import _append_python_callable_object_and_return_id - from .libpaddle import _cleanup, _Scope - from .libpaddle import _get_use_default_grad_op_desc_maker_ops - from .libpaddle import _get_all_register_op_kernels - from .libpaddle import _get_registered_phi_kernels - from .libpaddle import _is_program_version_supported - from .libpaddle import _set_eager_deletion_mode - from .libpaddle import _get_eager_deletion_vars - from .libpaddle import _set_fuse_parameter_group_size - from .libpaddle import _set_fuse_parameter_memory_size - from .libpaddle import _is_dygraph_debug_enabled - from .libpaddle import _dygraph_debug_level - from .libpaddle import _switch_tracer + from .libpaddle import ( + __doc__, + __file__, + __name__, + __package__, + ) # noqa: F401 + from .libpaddle import __unittest_throw_exception__ # noqa: F401 + from .libpaddle import ( + _append_python_callable_object_and_return_id, + ) # noqa: F401 + from .libpaddle import _cleanup, _Scope # noqa: F401 + from .libpaddle import _get_use_default_grad_op_desc_maker_ops # noqa: F401 + from .libpaddle import _get_all_register_op_kernels # noqa: F401 + from .libpaddle import _get_registered_phi_kernels # noqa: F401 + from .libpaddle import _is_program_version_supported # noqa: F401 + from .libpaddle import _set_eager_deletion_mode # noqa: F401 + from .libpaddle import _get_eager_deletion_vars # noqa: F401 + from .libpaddle import _set_fuse_parameter_group_size # noqa: F401 + from .libpaddle import _set_fuse_parameter_memory_size # noqa: F401 + from .libpaddle import _is_dygraph_debug_enabled # noqa: F401 + from .libpaddle import _dygraph_debug_level # noqa: F401 + from .libpaddle import _switch_tracer # noqa: F401 from .libpaddle import _set_paddle_lib_path - from .libpaddle import _create_loaded_parameter - from .libpaddle import _cuda_synchronize - from .libpaddle import _test_enforce_gpu_success - from .libpaddle import _is_compiled_with_heterps - from .libpaddle import _promote_types_if_complex_exists - from .libpaddle import _set_cached_executor_build_strategy - from .libpaddle import _device_synchronize - from .libpaddle import _xpu_device_synchronize - from .libpaddle import _get_current_stream - from .libpaddle import _Profiler, _ProfilerResult, _RecordEvent - from .libpaddle import _set_current_stream - from .libpaddle import _get_phi_kernel_name + from .libpaddle import _create_loaded_parameter # noqa: F401 + from .libpaddle import _cuda_synchronize # noqa: F401 + from .libpaddle import _test_enforce_gpu_success # noqa: F401 + from .libpaddle import _is_compiled_with_heterps # noqa: F401 + from .libpaddle import _promote_types_if_complex_exists # noqa: F401 + from .libpaddle import _set_cached_executor_build_strategy # noqa: F401 + from .libpaddle import _device_synchronize # noqa: F401 + from .libpaddle import _xpu_device_synchronize # noqa: F401 + from .libpaddle import _get_current_stream # noqa: F401 + from .libpaddle import ( + _Profiler, + _ProfilerResult, + _RecordEvent, + ) # noqa: F401 + from .libpaddle import _set_current_stream # noqa: F401 + from .libpaddle import _get_phi_kernel_name # noqa: F401 # prim controller flags from .libpaddle import __set_bwd_prim_enabled @@ -317,31 +327,33 @@ def to_list(s): from .libpaddle import __set_all_prim_enabled from .libpaddle import _is_eager_prim_enabled from .libpaddle import __set_eager_prim_enabled - from .libpaddle import _set_prim_target_grad_name - from .libpaddle import _add_skip_comp_ops + from .libpaddle import _set_prim_target_grad_name # noqa: F401 + from .libpaddle import _add_skip_comp_ops # noqa: F401 from .libpaddle import _set_bwd_prim_blacklist - from .libpaddle import _remove_skip_comp_ops + from .libpaddle import _remove_skip_comp_ops # noqa: F401 # custom devivce - from .libpaddle import _get_current_custom_device_stream - from .libpaddle import _set_current_custom_device_stream - from .libpaddle import _synchronize_custom_device - from .libpaddle import CustomDeviceStream - from .libpaddle import CustomDeviceEvent + from .libpaddle import _get_current_custom_device_stream # noqa: F401 + from .libpaddle import _set_current_custom_device_stream # noqa: F401 + from .libpaddle import _synchronize_custom_device # noqa: F401 + from .libpaddle import CustomDeviceStream # noqa: F401 + from .libpaddle import CustomDeviceEvent # noqa: F401 if sys.platform != 'win32': - from .libpaddle import _set_process_pids - from .libpaddle import _erase_process_pids - from .libpaddle import _set_process_signal_handler - from .libpaddle import _throw_error_if_process_failed - from .libpaddle import _convert_to_tensor_list - from .libpaddle import _array_to_share_memory_tensor - from .libpaddle import _cleanup_mmap_fds - from .libpaddle import _remove_tensor_list_mmap_fds - from .libpaddle import _set_max_memory_map_allocation_pool_size + from .libpaddle import _set_process_pids # noqa: F401 + from .libpaddle import _erase_process_pids # noqa: F401 + from .libpaddle import _set_process_signal_handler # noqa: F401 + from .libpaddle import _throw_error_if_process_failed # noqa: F401 + from .libpaddle import _convert_to_tensor_list # noqa: F401 + from .libpaddle import _array_to_share_memory_tensor # noqa: F401 + from .libpaddle import _cleanup_mmap_fds # noqa: F401 + from .libpaddle import _remove_tensor_list_mmap_fds # noqa: F401 + from .libpaddle import ( + _set_max_memory_map_allocation_pool_size, + ) # noqa: F401 # CINN - from .libpaddle import is_run_with_cinn + from .libpaddle import is_run_with_cinn # noqa: F401 except Exception as e: if has_paddle_dy_lib: diff --git a/python/paddle/base/data_feeder.py b/python/paddle/base/data_feeder.py index b8044dc2b7885..4e3545c1bcb8e 100644 --- a/python/paddle/base/data_feeder.py +++ b/python/paddle/base/data_feeder.py @@ -14,8 +14,6 @@ from . import core import numpy as np -import os -import multiprocessing import warnings import struct @@ -23,7 +21,7 @@ Variable, default_main_program, in_dygraph_mode, - _current_expected_place, + _current_expected_place, # noqa: F401 ) from .framework import _cpu_num, _cuda_ids diff --git a/python/paddle/base/device_worker.py b/python/paddle/base/device_worker.py index 4083937e2d76d..b5ea7dab355d3 100644 --- a/python/paddle/base/device_worker.py +++ b/python/paddle/base/device_worker.py @@ -623,7 +623,6 @@ def _gen_worker_desc(self, trainer_desc): Args: trainer_desc(TrainerDesc): a TrainerDesc object """ - from google.protobuf import text_format from . import core trainer_desc.device_worker_name = "SectionWorker" @@ -671,8 +670,6 @@ def _gen_worker_desc(self, trainer_desc): Args: trainer_desc(TrainerDesc): a TrainerDesc object """ - from google.protobuf import text_format - from . import core trainer_desc.device_worker_name = "HeterSectionWorker" heter_pipeline_opt = self._program._heter_pipeline_opt diff --git a/python/paddle/base/dygraph/math_op_patch.py b/python/paddle/base/dygraph/math_op_patch.py index 1a73e6b5e9ae0..81d9abdea25d4 100644 --- a/python/paddle/base/dygraph/math_op_patch.py +++ b/python/paddle/base/dygraph/math_op_patch.py @@ -13,18 +13,17 @@ # limitations under the License. from .. import core -from ..framework import ( +from ..framework import ( # noqa: F401 Variable, convert_np_dtype_to_dtype_, in_dygraph_mode, ) -from ..framework import _create_tensor as framework_create_tensor -from ..layers.layer_function_generator import OpProtoHolder -from . import no_grad -from .. import framework +from ..framework import _create_tensor as framework_create_tensor # noqa: F401 +from ..layers.layer_function_generator import OpProtoHolder # noqa: F401 +from . import no_grad # noqa: F401 +from .. import framework # noqa: F401 import numpy as np -import warnings from paddle import _C_ops, _legacy_C_ops _supported_int_dtype_ = [ diff --git a/python/paddle/base/dygraph/tensor_patch_methods.py b/python/paddle/base/dygraph/tensor_patch_methods.py index 2db290e1e1d4e..616be94913818 100644 --- a/python/paddle/base/dygraph/tensor_patch_methods.py +++ b/python/paddle/base/dygraph/tensor_patch_methods.py @@ -15,7 +15,6 @@ import inspect import numpy as np import warnings -import weakref import sys import paddle @@ -23,7 +22,7 @@ from ..framework import convert_np_dtype_to_dtype_ from .. import core from .. import unique_name -from ..framework import ( +from ..framework import ( # noqa: F401 Variable, Parameter, _getitem_static, @@ -32,8 +31,8 @@ EagerParamBase, in_dygraph_mode, ) -from .base import switch_to_static_graph -from .math_op_patch import monkey_patch_math_tensor +from .base import switch_to_static_graph # noqa: F401 +from .math_op_patch import monkey_patch_math_tensor # noqa: F401 from paddle.base.data_feeder import ( convert_uint16_to_float, _PADDLE_DTYPE_2_NUMPY_DTYPE, @@ -41,9 +40,9 @@ import paddle.utils.deprecated as deprecated import paddle.profiler as profiler from paddle.profiler.utils import in_profiler_mode -from paddle import _C_ops, _legacy_C_ops -from paddle.device import get_all_custom_device_type -from paddle.base.framework import _global_flags +from paddle import _C_ops, _legacy_C_ops # noqa: F401 +from paddle.device import get_all_custom_device_type # noqa: F401 +from paddle.base.framework import _global_flags # noqa: F401 _grad_scalar = None diff --git a/python/paddle/base/dygraph/tracer.py b/python/paddle/base/dygraph/tracer.py index 34493a0ffa90a..35cbe88f91f87 100644 --- a/python/paddle/base/dygraph/tracer.py +++ b/python/paddle/base/dygraph/tracer.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict - -import numpy as np from paddle.base import core from paddle.base import framework diff --git a/python/paddle/base/executor.py b/python/paddle/base/executor.py index 42abbd1b3b717..89395e6f79ac9 100755 --- a/python/paddle/base/executor.py +++ b/python/paddle/base/executor.py @@ -14,7 +14,6 @@ import logging import os -import multiprocessing import sys import warnings import numpy as np @@ -42,7 +41,7 @@ import copy from . import framework from .incubate.checkpoint import auto_checkpoint as acp -from .compiler import _prune_feed_ops +from .compiler import _prune_feed_ops # noqa: F401 from functools import lru_cache diff --git a/python/paddle/base/framework.py b/python/paddle/base/framework.py index 117912c4f1894..cc5734fd6c979 100644 --- a/python/paddle/base/framework.py +++ b/python/paddle/base/framework.py @@ -14,9 +14,7 @@ import textwrap import collections -from collections import defaultdict from collections.abc import Iterable -import contextlib from .wrapped_decorator import signature_safe_contextmanager, wrap_decorator import os import re @@ -28,9 +26,9 @@ import subprocess import multiprocessing import sys -import logging +import logging # noqa: F401 -from .proto import framework_pb2, data_feed_pb2 +from .proto import framework_pb2 from . import core from . import unique_name diff --git a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py index 7d07c4594acb4..93c2bc19ac06e 100644 --- a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py +++ b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py @@ -14,13 +14,10 @@ import sys import logging -import hashlib import json import os import time -import collections -from threading import Thread, current_thread -from contextlib import contextmanager +from threading import current_thread from paddle.base import unique_name, compiler from .checkpoint_saver import SerializableBase, CheckpointSaver, PaddleModel diff --git a/python/paddle/base/initializer.py b/python/paddle/base/initializer.py index c9d197a4f6840..c55a5f7e952e3 100644 --- a/python/paddle/base/initializer.py +++ b/python/paddle/base/initializer.py @@ -12,21 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import math -import functools -from . import framework -from . import core -from .framework import ( +from . import framework # noqa: F401 +from . import core # noqa: F401 +from .framework import ( # noqa: F401 in_dygraph_mode, default_main_program, _current_expected_place, ) -from .framework import program_guard -import numpy as np -from .core import VarDesc -from . import unique_name -from .data_feeder import check_variable_and_dtype, check_type, check_dtype -from paddle import _C_ops, _legacy_C_ops +from .framework import program_guard # noqa: F401 +from .core import VarDesc # noqa: F401 +from . import unique_name # noqa: F401 +from .data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, +) # noqa: F401 +from paddle import _C_ops, _legacy_C_ops # noqa: F401 import paddle __all__ = ['set_global_initializer'] diff --git a/python/paddle/base/io.py b/python/paddle/base/io.py index 80855333b9165..4a23bb49cc533 100644 --- a/python/paddle/base/io.py +++ b/python/paddle/base/io.py @@ -12,22 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import errno -import warnings import logging -import pickle -import contextlib -from functools import reduce -import sys -from io import BytesIO -import numpy as np -import math -import paddle -from paddle.base import layers -from paddle.base.executor import Executor, global_scope -from paddle.base.framework import ( +import numpy as np # noqa: F401 +from paddle.base import layers # noqa: F401 +from paddle.base.executor import Executor, global_scope # noqa: F401 +from paddle.base.framework import ( # noqa: F401 Program, Parameter, default_main_program, @@ -37,7 +27,7 @@ dygraph_not_support, static_only, ) -from paddle.reader import ( +from paddle.reader import ( # noqa: F401 cache, map_readers, buffered, @@ -49,15 +39,15 @@ xmap_readers, multiprocess_reader, ) -from .wrapped_decorator import signature_safe_contextmanager -from paddle.base.compiler import CompiledProgram -from paddle.base.log_helper import get_logger -from . import reader -from . import unique_name -from .reader import * -from . import core -from paddle.utils import deprecated -from paddle.base.framework import static_only +from .wrapped_decorator import signature_safe_contextmanager # noqa: F401 +from paddle.base.compiler import CompiledProgram # noqa: F401 +from paddle.base.log_helper import get_logger # noqa: F401 +from . import reader # noqa: F401 +from . import unique_name # noqa: F401 +from .reader import * # noqa: F401 +from . import core # noqa: F401 +from paddle.utils import deprecated # noqa: F401 +from paddle.base.framework import static_only # noqa: F401 __all__ = reader.__all__ diff --git a/python/paddle/base/layer_helper.py b/python/paddle/base/layer_helper.py index 96722b231acb5..dd97cf62dfebf 100644 --- a/python/paddle/base/layer_helper.py +++ b/python/paddle/base/layer_helper.py @@ -18,12 +18,10 @@ Parameter, dtype_is_floating, in_dygraph_mode, - OpProtoHolder, _global_flags, ) from . import unique_name from .param_attr import ParamAttr -from . import core from .layer_helper_base import LayerHelperBase from .dygraph_utils import _append_activation_in_dygraph diff --git a/python/paddle/base/layers/layer_function_generator.py b/python/paddle/base/layers/layer_function_generator.py index 358c2de238de1..4ce87e047667a 100644 --- a/python/paddle/base/layers/layer_function_generator.py +++ b/python/paddle/base/layers/layer_function_generator.py @@ -13,7 +13,6 @@ # limitations under the License. import re -import functools import warnings import string diff --git a/python/paddle/base/reader.py b/python/paddle/base/reader.py index 001c96bd7df26..5adec86da108a 100644 --- a/python/paddle/base/reader.py +++ b/python/paddle/base/reader.py @@ -17,26 +17,20 @@ import numpy as np import threading import paddle -import time -import copy from .framework import ( Program, - Variable, program_guard, default_main_program, default_startup_program, in_dygraph_mode, - cpu_places, _current_expected_place, ) from .executor import global_scope from .data_feeder import DataFeeder, BatchedTensorProvider from .multiprocess_utils import ( - multiprocess_queue_set, CleanupFuncRegistrar, _cleanup_mmap, - _cleanup, _set_SIGCHLD_handler, ) from .layers.io import ( @@ -51,9 +45,7 @@ import warnings ### Dygraph DataLoader configs ### -import os import multiprocessing -import signal import queue diff --git a/python/paddle/base/trainer_desc.py b/python/paddle/base/trainer_desc.py index 9820f2cca0312..48cc427ac8e7e 100644 --- a/python/paddle/base/trainer_desc.py +++ b/python/paddle/base/trainer_desc.py @@ -305,8 +305,6 @@ def _set_copy_table_config(self, config_dict): ) def _desc(self): - from google.protobuf import text_format - return self.proto_desc.SerializeToString() def __str__(self): diff --git a/python/paddle/base/trainer_factory.py b/python/paddle/base/trainer_factory.py index 859d8c27f71d2..2a4b689aaf0b0 100644 --- a/python/paddle/base/trainer_factory.py +++ b/python/paddle/base/trainer_factory.py @@ -25,22 +25,11 @@ from .trainer_desc import ( MultiTrainer, - DistMultiTrainer, - PipelineTrainer, - HeterXpuTrainer, - PSGPUTrainer, - HeterPipelineTrainer, ) from .device_worker import ( Hogwild, - DownpourSGD, - DownpourLite, - Section, - DownpourSGDOPT, - HeterSection, ) from .framework import Variable -from multiprocessing import Process, Manager __all__ = ["TrainerFactory", "FetchHandlerMonitor"] diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index ac0215b32d59b..b6cd1a01ee48a 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import numpy as np from . import unique_name from . import core @@ -373,7 +372,6 @@ def _setitem_for_tensor_array(var, item, value): from paddle.jit.dy2static.variable_trans_func import ( to_static_variable, ) - from paddle import cast from paddle.tensor import array_write item = paddle.cast(to_static_variable(item), dtype='int64') diff --git a/test/legacy_test/test_math_op_patch.py b/test/legacy_test/test_math_op_patch.py index 55b3f55c45451..004b61668e0e6 100644 --- a/test/legacy_test/test_math_op_patch.py +++ b/test/legacy_test/test_math_op_patch.py @@ -14,7 +14,6 @@ import unittest -import numpy import numpy as np from decorator_helper import prog_scope diff --git a/test/legacy_test/test_rnn_cell_api.py b/test/legacy_test/test_rnn_cell_api.py index cf83bdb8120ec..58494f5415830 100644 --- a/test/legacy_test/test_rnn_cell_api.py +++ b/test/legacy_test/test_rnn_cell_api.py @@ -15,7 +15,6 @@ import sys import unittest -import numpy import numpy as np sys.path.append("../../test/rnn")