Merge branch 'develop' into support

Author: DanielSun11

.github/workflows/Api-Benchmark-baseline.yml

@@ -18,7 +18,6 @@ on:
           - others
   schedule:
     - cron: '0 21 * * *'
-    - cron: '0 22 * * 3'
 
 permissions: read-all
 
@@ -52,7 +51,7 @@ jobs:
       is_pr: 'false'
 
   api-benchmark-baseline-schedule:
-    name: Api benchmark baseline
+    name: Api benchmark baseline with schedule
     strategy:
       matrix:
         run-labels: [api-bm-20, api-bm-27]
@@ -63,49 +62,26 @@ jobs:
       baseline: 'true'
       run-labels: ${{ matrix.run-labels }}
 
-  api-benchmark-baseline-pr:
-    name: Api benchmark baseline
+  api-benchmark-baseline-pr-20:
+    name: Api benchmark baseline with PR on 20
     if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
-    strategy:
-      matrix:
-        run-labels: [api-bm-20, api-bm-27]
     uses: ./.github/workflows/_Api-Benchmark.yml
     needs: [clone, build-docker]
     with:
       docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
       baseline: 'true'
       MANUALLY_PR_ID: ${{ inputs.PR_ID }}
       MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
-      run-labels: ${{ matrix.run-labels }}
+      run-labels: api-bm-20
 
-  test1:
-    runs-on: ubuntu-latest
-    if: github.event.schedule == '0 0 * * *'
-    steps:
-      - name: Test
-        run: |
-          echo "test1"
-
-  test2:
-    runs-on: ubuntu-latest
-    if: github.event.schedule == '0 21 * * *'
-    steps:
-      - name: Test
-        run: |
-          echo "test2"
-
-  test3:
-    runs-on: ubuntu-latest
-    if: github.event.schedule == '0 22 * * 3'
-    steps:
-      - name: Test
-        run: |
-          echo "test3"
-
-  test4:
-    runs-on: ubuntu-latest
-    if: github.event.schedule == '0 21 * * 1'
-    steps:
-      - name: Test
-        run: |
-          echo "test4"
+  api-benchmark-baseline-pr-27:
+    name: Api benchmark baseline with PR on 27
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
+    uses: ./.github/workflows/_Api-Benchmark.yml
+    needs: [clone, build-docker]
+    with:
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+      baseline: 'true'
+      MANUALLY_PR_ID: ${{ inputs.PR_ID }}
+      MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
+      run-labels: api-bm-27

.github/workflows/_Api-Benchmark.yml

@@ -31,7 +31,7 @@ env:
   PADDLE_ROOT: /paddle
   TASK: paddle-CI-${{ github.event.pull_request.number }}-api-benchmark
   ci_scripts: /paddle/ci
-  BRANCH: ${{ github.event.pull_request.base.ref }}
+  BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
   CI_name: api-benchmark
   no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
@@ -76,7 +76,7 @@ jobs:
       - name: Check docker image and run container
         env:
           python: "python3.10"
-          GIT_PR_ID: ${{ github.event.pull_request.number }}
+          GIT_PR_ID: ${{ github.event.pull_request.number || '0' }}
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUN_ID: ${{ github.run_id }}
           wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
@@ -134,13 +134,14 @@ jobs:
           cp /paddle/PTSTools/Uploader/apibm_config.yml .
           source ${{ github.workspace }}/../../../proxy
           if [[ "${{ inputs.baseline }}" == "true" ]];then
+            set -e
             if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
-              export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+              export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/$PR_ID/$COMMIT_ID/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
             else
               export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
             fi
             ${python} -m pip install $pr_wheel_link
-            ${python} runner_ci_action.py --yaml ../yaml/api_benchmark_fp32.yml --baseline_whl_link $pr_wheel_link
+            ${python} runner_ci_multipro_action.py --yaml ../yaml/sort_api_benchmark_fp32.yml --core_index ${core_index} --baseline_whl_link $pr_wheel_link
             exit 0
           fi
           ${python} -m pip install $wheel_link

.pre-commit-config.yaml

@@ -55,116 +55,12 @@ repos:
       - id: typos
         args: [--force-exclude]
   # For Python files
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.1.0
-    hooks:
-      - id: black
-        exclude: |
-          (?x)^(
-            ci/.+
-
-            | cmake/.+
-
-            | r/.+
-
-            | paddle/scripts/.+
-
-            | setup.py
-
-            | paddle/.+
-
-            | python/paddle/[a-c].+
-
-            | python/paddle/de.+
-
-            | python/paddle/distributed/a.+
-
-            | python/paddle/distributed/[b-e].+
-
-            # | python/paddle/distributed/f.+
-
-            # | python/paddle/distributed/[g-z].+
-
-            # | python/paddle/[e-i].+
-
-            # | python/paddle/j.+
-
-            # | python/paddle/[k-n].+
-
-            # | python/paddle/[o-t].+
-
-            # | python/paddle/[u-z].+
-
-            # | python/_.+
-
-            # | test/a.+
-
-            # | test/[b-h].+
-
-            # | test/[i-k].+
-
-            # | test/l.+
-
-            # | test/[m-z].+
-
-            # | tools/.+
-          )$
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.12.0
     hooks:
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix, --no-cache]
       - id: ruff-format
-        exclude: |
-          (?x)^(
-            # ci/.+
-
-            # | cmake/.+
-
-            # | r/.+
-
-            # | paddle/scripts/.+
-
-            # | setup.py
-
-            # | paddle/.+
-
-            # | python/paddle/[a-c].+
-
-            # | python/paddle/de.+
-
-            # | python/paddle/distributed/a.+
-
-            # | python/paddle/distributed/[b-e].+
-
-            | python/paddle/distributed/f.+
-
-            | python/paddle/distributed/[g-z].+
-
-            | python/paddle/[e-i].+
-
-            | python/paddle/j.+
-
-            | python/paddle/[k-n].+
-
-            | python/paddle/[o-t].+
-
-            | python/paddle/[u-z].+
-
-            | python/_.+
-
-            | test/a.+
-
-            | test/[b-h].+
-
-            | test/[i-k].+
-
-            | test/l.+
-
-            | test/[m-z].+
-
-            | tools/.+
-          )$
   # For C++ files
   - repo: local
     hooks:

ci/auto_parallel/ci_auto_parallel.sh

@@ -77,6 +77,7 @@ get_diff_TO_case(){
     case_list[${#case_list[*]}]=llama_auto
     case_list[${#case_list[*]}]=gpt-3_auto
     case_list[${#case_list[*]}]=gpt-3_dygraph
+    case_list[${#case_list[*]}]=deepseek_auto
 }
 
 print_info(){
@@ -258,6 +259,14 @@ if [[ ${#case_list[*]} -ne 0 ]];then
             execute_func_list $cmd gpt-3_dygraph
             let case_num++
             clean_file ${work_dir}/../PaddleNLP/llm
+        elif [[ ${case} == "deepseek_auto" ]];then
+            cmd=${work_dir}/../PaddleNLP/scripts/distribute/ci_case_auto.sh
+            timeout 5m bash $cmd prepare_case deepseek_case_list_auto $FLAGS_install_deps $FLAGS_download_data
+            execute_func_list $cmd deepseek_auto
+            export FLAGS_install_deps=1
+            export FLAGS_download_data="deepseek ""$FLAGS_download_data"
+            let case_num++
+            clean_file ${work_dir}/../PaddleNLP/llm/auto_parallel/deepseek-v3
         else
             echo -e "\033[31m ---- no ${case} \033"
             let case_num++

ci/coverage_test.sh

@@ -24,10 +24,14 @@ function is_run_distribute_in_op_test() {
             echo "export FLAGS_COVERAGE_RUN_AUTO_PARALLEL_IN_OP_TEST=1" >> "$HOME/.bashrc"
         fi
     done
-    ALL_CHANGE_FILES=`git diff --numstat upstream/$BRANCH | awk '{print $3}' | grep ".py"|| true`
+    ALL_CHANGE_FILES=$(git diff --name-only upstream/$BRANCH | grep ".py"|| true)
     echo ${ALL_CHANGE_FILES}
     for CHANGE_FILE in ${ALL_CHANGE_FILES}; do
-        ALL_OPTEST_BAN_AUTO_PARALLEL_TEST=`git diff -U0 upstream/$BRANCH ${PADDLE_ROOT}/${CHANGE_FILE} | grep "+" | grep "check_auto_parallel=" || true`
+        TARGET_FILE="${PADDLE_ROOT}/${CHANGE_FILE}"
+        if [ ! -f "$TARGET_FILE" ]; then
+            continue
+        fi
+        ALL_OPTEST_BAN_AUTO_PARALLEL_TEST=`git diff -U0 upstream/$BRANCH "TARGET_FILE" | grep "+" | grep "check_auto_parallel=" || true`
         if [ "${ALL_OPTEST_BAN_AUTO_PARALLEL_TEST}" != "" ] && [ "${GIT_PR_ID}" != "" ]; then
             export FLAGS_COVERAGE_RUN_AUTO_PARALLEL_IN_OP_TEST=1
             echo "export FLAGS_COVERAGE_RUN_AUTO_PARALLEL_IN_OP_TEST=1" >> "$HOME/.bashrc"

paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py

@@ -525,10 +525,8 @@ def __init__(self, forward_api_contents, namespace):
         self.dygraph_pre_process = (
             ""  # The pre_process function calling code for dygraph
         )
-        self.static_pre_process = (
-            ""  # The pre_process function calling code for static graph
-        )
-        self.args_parser_func_name = ""  # The custom args parser function
+
+        self.args_mapper_func_name = None  # The custom args parser function
         self.python_api_names = ""
 
     def ParseForwardInplaceInfo(self):
@@ -561,20 +559,19 @@ def ParsePythonAPIInfo(self):
             self.args_alias_map = args_alias
         if 'pre_process' in python_api_info.keys():
             pre_process = python_api_info['pre_process']
-            if 'func' in pre_process.keys():
-                self.dygraph_pre_process = pre_process['func']
-                self.static_pre_process = pre_process['func']
-                # TODO check len(pre_process) > 1
-
-            if 'dygraph_func' in pre_process.keys():
-                self.dygraph_pre_process = pre_process['dygraph_func']
-            if 'static_func' in pre_process.keys():
-                self.static_pre_process = pre_process['static_func']
-        if (
-            'args_parser' in python_api_info.keys()
-            and 'func' in python_api_info['args_parser']
-        ):
-            self.args_parser_func_name = python_api_info['args_parser']['func']
+            if pre_process is not None:
+                if 'dygraph_func' in pre_process.keys():
+                    self.dygraph_pre_process = pre_process['dygraph_func']
+                elif 'func' in pre_process.keys():
+                    self.dygraph_pre_process = pre_process['func']
+
+        if 'args_mapper' in python_api_info.keys():
+            args_mapper = python_api_info['args_mapper']
+            if args_mapper is not None:
+                if 'dygraph_func' in args_mapper.keys():
+                    self.args_mapper_func_name = args_mapper['dygraph_func']
+                elif 'func' in args_mapper.keys():
+                    self.args_mapper_func_name = args_mapper['func']
 
     def ParseNoNeedBuffer(self):
         grad_api_contents = self.grad_api_contents