From cb2ff1aa1c8cfb3383c101c3b9bc21a9bce54bfb Mon Sep 17 00:00:00 2001 From: yangrongxinuser <109195068+yangrongxinuser@users.noreply.github.com> Date: Thu, 5 Dec 2024 19:04:18 +0800 Subject: [PATCH 1/2] revise --- .pre-commit-config.yaml | 214 +++++++++++++++++++++++-------- docs/zh/examples/cfdgcn.md | 16 ++- examples/cfdgcn/cfdgcn.py | 92 ++++++++++++- examples/cfdgcn/conf/cfdgcn.yaml | 14 ++ 4 files changed, 281 insertions(+), 55 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33fb9870f..43ecca818 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,56 +1,164 @@ +# Exclude all third-party libraries and auto-generated files globally +exclude: | + (?x)^( + patches/.+| + paddle/fluid/framework/fleet/heter_ps/cudf/.+| + paddle/fluid/distributed/ps/thirdparty/round_robin.h| + python/paddle/utils/gast/.+| + third_party/.+ + )$ repos: - - repo: https://github.com/PyCQA/isort - rev: 5.11.5 - hooks: - - id: isort - args: ["--multi-line=7", "--sl", "--profile", "black", "--filter-files"] - - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.0.272" - hooks: - - id: ruff - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: a11d9314b22d8f8c7556443875b731ef05965464 - hooks: - - id: check-merge-conflict - - id: check-symlinks - - id: detect-private-key - files: (?!.*paddle)^.*$ - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict - - id: check-yaml - exclude: "mkdocs.yml|recipe/meta.yaml" - - id: pretty-format-json - args: [--autofix] - - id: requirements-txt-fixer - - - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.0.1 - hooks: - - id: forbid-crlf - files: \.md$ - - id: remove-crlf - files: \.md$ - - id: forbid-tabs - files: \.md$ - - id: remove-tabs - files: \.md$ - - - repo: local - hooks: - - id: clang-format +# Common hooks +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: sort-simple-yaml + files: (ops|backward|op_[a-z_]+)\.yaml$ + - id: trailing-whitespace +- repo: https://github.com/Lucas-C/pre-commit-hooks.git + rev: v1.5.1 + hooks: + - id: remove-crlf + - id: remove-tabs + name: Tabs remover (C++) + files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$ + args: [--whitespaces-count, '2'] + - id: remove-tabs + name: Tabs remover (Python) + files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ + args: [--whitespaces-count, '4'] + # Exclude some unit test files that require tabs. + exclude: | + (?x)^( + test/dygraph_to_static/test_error.py + )$ +- repo: local + hooks: + - id: copyright_checker + name: copyright_checker + entry: python ./tools/codestyle/copyright.py + language: system + files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|pyi|sh)$ + exclude: | + (?x)^( + paddle/utils/.*| + paddle/cinn/utils/registry.h + )$ +- repo: https://github.com/PFCCLab/typos-pre-commit-mirror.git + rev: v1.27.3 + hooks: + - id: typos + args: [--force-exclude] +# For Python files +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.1 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --no-cache] +# For C++ files +- repo: local + hooks: + - id: clang-format name: clang-format - description: Format files with ClangFormat - entry: bash .clang_format.hook -i + description: Format files with ClangFormat. + entry: bash ./tools/codestyle/clang_format.sh -i language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$ - -exclude: | - ^jointContribution/ + files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$ +- repo: local + hooks: + - id: cpplint-cpp-source + name: cpplint + description: Check C++ code style using cpplint.py. + entry: bash ./tools/codestyle/cpplint_pre_commit.sh + language: system + files: \.(cc|cxx|cpp|cu|h|hpp|hxx)$ + args: + - --extensions=cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps + - --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11,-whitespace/parens + - --quiet + # Exclude third-party libraries + exclude: | + (?x)^( + paddle/utils/flat_hash_map\.h + )$ +- repo: local + hooks: + - id: clang-tidy + name: clang-tidy + description: Parallel clang-tidy runner. + entry: python ./tools/codestyle/clang-tidy.py + language: system + files: \.(c|cc|cxx|cpp|h|hpp|hxx)$ + args: + - -p=build/ + - -extra-arg=-Wno-unknown-warning-option + - -extra-arg=-Wno-pessimizing-move + - -extra-arg=-Wno-braced-scalar-init + - -extra-arg=-Wno-dangling-gsl + - -extra-arg=-Wno-deprecated-copy + - -extra-arg=-Wno-final-dtor-non-final-class + - -extra-arg=-Wno-implicit-int-float-conversion + - -extra-arg=-Wno-inconsistent-missing-override + - -extra-arg=-Wno-infinite-recursion + - -extra-arg=-Wno-mismatched-tags + - -extra-arg=-Wno-self-assign + - -extra-arg=-Wno-sign-compare + - -extra-arg=-Wno-sometimes-uninitialized + - -extra-arg=-Wno-tautological-overlap-compare + - -extra-arg=-Wno-unused-const-variable + - -extra-arg=-Wno-unused-lambda-capture + - -extra-arg=-Wno-unused-private-field + - -extra-arg=-Wno-unused-value + - -extra-arg=-Wno-unused-variable + - -extra-arg=-Wno-overloaded-virtual + - -extra-arg=-Wno-defaulted-function-deleted + - -extra-arg=-Wno-delete-non-abstract-non-virtual-dtor + - -extra-arg=-Wno-return-type-c-linkage +# For CMake files +- repo: local + hooks: + - id: auto-generate-cmakelists + name: auto-generate-cmakelists + entry: bash ./tools/gen_ut_cmakelists.hook + language: system + files: testslist.csv$ +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.13 + hooks: + - id: cmake-format + # exclude paddle/fluid/operators/CMakeLists.txt, see the comment + # https://github.com/PaddlePaddle/Paddle/pull/43057#pullrequestreview-993471860 + exclude: | + (?x)^( + paddle/fluid/operators/CMakeLists.txt + )$ +- repo: https://github.com/PFCCLab/cmake-lint-paddle + rev: v1.5.1 + hooks: + - id: cmakelint + args: [--config=./tools/codestyle/.cmakelintrc] + # Exclude some files has false positive warnings + # Need to fix them in the future + exclude: | + (?x)^( + cmake/external/onnxruntime.cmake + )$ +# Others +- repo: local + hooks: + - id: sort-txt-file + name: sort-txt-file + description: Sorts each line string in a text file + entry: python ./tools/codestyle/sort_txt_file.py + language: python + files: test/white_list/pir_op_test_white_list + args: [] diff --git a/docs/zh/examples/cfdgcn.md b/docs/zh/examples/cfdgcn.md index a8374110b..59cba96ab 100644 --- a/docs/zh/examples/cfdgcn.md +++ b/docs/zh/examples/cfdgcn.md @@ -2,6 +2,20 @@ AI Studio快速体验 +=== "模型导出命令" + + ``` sh + python cfdgcn.py mode=export + ``` + + +=== "模型推理命令" + + ``` sh + python cfdgcn.py mode=infer + ``` + + === "模型训练命令" ``` sh @@ -23,7 +37,7 @@ # generalization experiments mpirun -np $((BATCH_SIZE+1)) python cfdgcn.py \ TRAIN.batch_size=$((BATCH_SIZE)) \ - TRAIN_DATA_DIR="./data/NACA0012_machsplit_noshock/outputs_train" \ + TRAIN_DATA_DIR="./data/NACA0012_machsplit_noshock/outputs_train" \ TRAIN_MESH_GRAPH_PATH="./data/NACA0012_machsplit_noshock/mesh_fine. su2" \ EVAL_DATA_DIR="./data/NACA0012_machsplit_noshock/outputs_test" \ EVAL_MESH_GRAPH_PATH="./data/NACA0012_machsplit_noshock/mesh_fine.su2" \ diff --git a/examples/cfdgcn/cfdgcn.py b/examples/cfdgcn/cfdgcn.py index d06d9c0c9..89092972c 100644 --- a/examples/cfdgcn/cfdgcn.py +++ b/examples/cfdgcn/cfdgcn.py @@ -157,6 +157,90 @@ def train(cfg: DictConfig): ) +def export(cfg: DictConfig): + # set model + model = ppsci.arch.CFDGCN( + **cfg.MODEL, + process_sim=process_sim, + fine_marker_dict=fine_marker_dict, + su2_module=su2paddle.SU2Module, + ) + + solver = ppsci.solver.Solver( + model, pretrained_model_path=cfg.EXPORT.pretrained_model_path + ) + + # export model + from paddle.static import InputSpec + + input_spec = [ + {key: InputSpec([None, 1], "float32", name=key) for key in model.input_keys}, + ] + + solver.export(input_spec, cfg.INFER.export_path) + + +def inference(cfg: DictConfig): + + # 初始化预测器 + from deploy.python_infer import pinn_predictor + + predictor = pinn_predictor.PINNPredictor(cfg) + + # 设置 dataloader 配置 + infer_dataloader_cfg = { + "dataset": { + "name": "MeshAirfoilDataset", + "input_keys": ("input",), + "label_keys": ("label",), + "data_dir": cfg.INFER_DATA_DIR, + "mesh_graph_path": cfg.INFER_MESH_GRAPH_PATH, + "transpose_edges": True, + }, + "batch_size": cfg.INFER.batch_size, + "sampler": { + "name": "BatchSampler", + "drop_last": False, + "shuffle": False, + }, + } + + # 初始化数据集和 dataloader + infer_dataset = ppsci.data.MeshAirfoilDataset( + input_keys=infer_dataloader_cfg["dataset"]["input_keys"], + label_keys=infer_dataloader_cfg["dataset"]["label_keys"], + data_dir=infer_dataloader_cfg["dataset"]["data_dir"], + mesh_graph_path=infer_dataloader_cfg["dataset"]["mesh_graph_path"], + transpose_edges=infer_dataloader_cfg["dataset"]["transpose_edges"], + ) + infer_dataloader = ppsci.dataloader.DataLoader( + infer_dataset, + batch_size=infer_dataloader_cfg["batch_size"], + shuffle=infer_dataloader_cfg["sampler"]["shuffle"], + drop_last=infer_dataloader_cfg["sampler"]["drop_last"], + num_workers=infer_dataloader_cfg.get("num_workers", 1), + ) + + # 进行推理并可视化结果 + with predictor.no_grad_context_manager(True): + for index, (input_dict, label_dict, _) in enumerate(infer_dataloader): + # 获取真实值 + truefield = label_dict["label"].y + # 模型预测 + output_dict = predictor.predict(input_dict, cfg.INFER.batch_size) + prefield = output_dict["pred"] + + # 可视化结果 + utils.log_images( + input_dict["input"].pos, + prefield, + truefield, + infer_dataset.elems_list, + index, + "cylinder", + ) + + def evaluate(cfg: DictConfig): # set dataloader config train_dataloader_cfg = { @@ -257,8 +341,14 @@ def main(cfg: DictConfig): train(cfg) elif cfg.mode == "eval": evaluate(cfg) + elif cfg.mode == "export": + export(cfg) + elif cfg.mode == "infer": + inference(cfg) else: - raise ValueError(f"cfg.mode should in ['train', 'eval'], but got '{cfg.mode}'") + raise ValueError( + f"cfg.mode should in ['train', 'eval', 'export', 'infer'], but got '{cfg.mode}'" + ) if __name__ == "__main__": diff --git a/examples/cfdgcn/conf/cfdgcn.yaml b/examples/cfdgcn/conf/cfdgcn.yaml index ad55a9e14..eac8a01a1 100644 --- a/examples/cfdgcn/conf/cfdgcn.yaml +++ b/examples/cfdgcn/conf/cfdgcn.yaml @@ -62,3 +62,17 @@ EVAL: batch_size: 1 pretrained_model_path: null eval_with_no_grad: true + +# inference settings +INFER: + pretrained_model_path: "https://paddle-org.bj.bcebos.com/paddlescience/models/cfdgcn/cfdgcn_pretrained.pdparams" + device: gpu + engine: native + precision: fp32 + ir_optim: true + min_subgraph_size: 5 + gpu_mem: 2000 + gpu_id: 0 + max_batch_size: 512 + num_cpu_threads: 4 + batch_size: 512 From 89129ca9666007b905f93f56e302fe1a5cdbdb39 Mon Sep 17 00:00:00 2001 From: yangrongxinuser <109195068+yangrongxinuser@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:00:41 +0800 Subject: [PATCH 2/2] update --- .pre-commit-config.yaml | 214 ++++++++++------------------------------ 1 file changed, 53 insertions(+), 161 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43ecca818..bb6ad2fe3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,164 +1,56 @@ -# Exclude all third-party libraries and auto-generated files globally -exclude: | - (?x)^( - patches/.+| - paddle/fluid/framework/fleet/heter_ps/cudf/.+| - paddle/fluid/distributed/ps/thirdparty/round_robin.h| - python/paddle/utils/gast/.+| - third_party/.+ - )$ repos: -# Common hooks -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-added-large-files - - id: check-merge-conflict - - id: check-symlinks - - id: detect-private-key - - id: end-of-file-fixer - - id: sort-simple-yaml - files: (ops|backward|op_[a-z_]+)\.yaml$ - - id: trailing-whitespace -- repo: https://github.com/Lucas-C/pre-commit-hooks.git - rev: v1.5.1 - hooks: - - id: remove-crlf - - id: remove-tabs - name: Tabs remover (C++) - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$ - args: [--whitespaces-count, '2'] - - id: remove-tabs - name: Tabs remover (Python) - files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ - args: [--whitespaces-count, '4'] - # Exclude some unit test files that require tabs. - exclude: | - (?x)^( - test/dygraph_to_static/test_error.py - )$ -- repo: local - hooks: - - id: copyright_checker - name: copyright_checker - entry: python ./tools/codestyle/copyright.py - language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|pyi|sh)$ - exclude: | - (?x)^( - paddle/utils/.*| - paddle/cinn/utils/registry.h - )$ -- repo: https://github.com/PFCCLab/typos-pre-commit-mirror.git - rev: v1.27.3 - hooks: - - id: typos - args: [--force-exclude] -# For Python files -- repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 - hooks: - - id: black -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.1 - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix, --no-cache] -# For C++ files -- repo: local - hooks: - - id: clang-format + - repo: https://github.com/PyCQA/isort + rev: 5.11.5 + hooks: + - id: isort + args: ["--multi-line=7", "--sl", "--profile", "black", "--filter-files"] + + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: "v0.0.272" + hooks: + - id: ruff + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: a11d9314b22d8f8c7556443875b731ef05965464 + hooks: + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + files: (?!.*paddle)^.*$ + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + - id: check-yaml + exclude: "mkdocs.yml|recipe/meta.yaml" + - id: pretty-format-json + args: [--autofix] + - id: requirements-txt-fixer + + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.0.1 + hooks: + - id: forbid-crlf + files: \.md$ + - id: remove-crlf + files: \.md$ + - id: forbid-tabs + files: \.md$ + - id: remove-tabs + files: \.md$ + + - repo: local + hooks: + - id: clang-format name: clang-format - description: Format files with ClangFormat. - entry: bash ./tools/codestyle/clang_format.sh -i + description: Format files with ClangFormat + entry: bash .clang_format.hook -i language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$ -- repo: local - hooks: - - id: cpplint-cpp-source - name: cpplint - description: Check C++ code style using cpplint.py. - entry: bash ./tools/codestyle/cpplint_pre_commit.sh - language: system - files: \.(cc|cxx|cpp|cu|h|hpp|hxx)$ - args: - - --extensions=cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps - - --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11,-whitespace/parens - - --quiet - # Exclude third-party libraries - exclude: | - (?x)^( - paddle/utils/flat_hash_map\.h - )$ -- repo: local - hooks: - - id: clang-tidy - name: clang-tidy - description: Parallel clang-tidy runner. - entry: python ./tools/codestyle/clang-tidy.py - language: system - files: \.(c|cc|cxx|cpp|h|hpp|hxx)$ - args: - - -p=build/ - - -extra-arg=-Wno-unknown-warning-option - - -extra-arg=-Wno-pessimizing-move - - -extra-arg=-Wno-braced-scalar-init - - -extra-arg=-Wno-dangling-gsl - - -extra-arg=-Wno-deprecated-copy - - -extra-arg=-Wno-final-dtor-non-final-class - - -extra-arg=-Wno-implicit-int-float-conversion - - -extra-arg=-Wno-inconsistent-missing-override - - -extra-arg=-Wno-infinite-recursion - - -extra-arg=-Wno-mismatched-tags - - -extra-arg=-Wno-self-assign - - -extra-arg=-Wno-sign-compare - - -extra-arg=-Wno-sometimes-uninitialized - - -extra-arg=-Wno-tautological-overlap-compare - - -extra-arg=-Wno-unused-const-variable - - -extra-arg=-Wno-unused-lambda-capture - - -extra-arg=-Wno-unused-private-field - - -extra-arg=-Wno-unused-value - - -extra-arg=-Wno-unused-variable - - -extra-arg=-Wno-overloaded-virtual - - -extra-arg=-Wno-defaulted-function-deleted - - -extra-arg=-Wno-delete-non-abstract-non-virtual-dtor - - -extra-arg=-Wno-return-type-c-linkage -# For CMake files -- repo: local - hooks: - - id: auto-generate-cmakelists - name: auto-generate-cmakelists - entry: bash ./tools/gen_ut_cmakelists.hook - language: system - files: testslist.csv$ -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: v0.6.13 - hooks: - - id: cmake-format - # exclude paddle/fluid/operators/CMakeLists.txt, see the comment - # https://github.com/PaddlePaddle/Paddle/pull/43057#pullrequestreview-993471860 - exclude: | - (?x)^( - paddle/fluid/operators/CMakeLists.txt - )$ -- repo: https://github.com/PFCCLab/cmake-lint-paddle - rev: v1.5.1 - hooks: - - id: cmakelint - args: [--config=./tools/codestyle/.cmakelintrc] - # Exclude some files has false positive warnings - # Need to fix them in the future - exclude: | - (?x)^( - cmake/external/onnxruntime.cmake - )$ -# Others -- repo: local - hooks: - - id: sort-txt-file - name: sort-txt-file - description: Sorts each line string in a text file - entry: python ./tools/codestyle/sort_txt_file.py - language: python - files: test/white_list/pir_op_test_white_list - args: [] + files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$ + +exclude: | + ^jointContribution/ \ No newline at end of file