Merge branch 'main' into add_siglip_fast_tokenizer_bis

.circleci/config.yml

@@ -34,64 +34,44 @@ jobs:
  - run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
  - run: mkdir -p test_preparation
  - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
- - store_artifacts:
- path: ~/transformers/tests_fetched_summary.txt
- - run: |
- if [ -f test_list.txt ]; then
- cp test_list.txt test_preparation/test_list.txt
- else
- touch test_preparation/test_list.txt
- fi
- - run: |
- if [ -f examples_test_list.txt ]; then
- mv examples_test_list.txt test_preparation/examples_test_list.txt
- else
- touch test_preparation/examples_test_list.txt
- fi
- - run: |
- if [ -f filtered_test_list_cross_tests.txt ]; then
- mv filtered_test_list_cross_tests.txt test_preparation/filtered_test_list_cross_tests.txt
- else
- touch test_preparation/filtered_test_list_cross_tests.txt
- fi
- - run: |
- if [ -f doctest_list.txt ]; then
- cp doctest_list.txt test_preparation/doctest_list.txt
- else
- touch test_preparation/doctest_list.txt
- fi
- - run: |
- if [ -f test_repo_utils.txt ]; then
- mv test_repo_utils.txt test_preparation/test_repo_utils.txt
- else
- touch test_preparation/test_repo_utils.txt
- fi
  - run: python utils/tests_fetcher.py --filter_tests
+ - run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
  - run: |
- if [ -f test_list.txt ]; then
- mv test_list.txt test_preparation/filtered_test_list.txt
- else
- touch test_preparation/filtered_test_list.txt
+ if [ ! -s test_preparation/generated_config.yml ]; then
+ echo "No tests to run, exiting early!"
+ circleci-agent step halt
  fi
+
  - store_artifacts:
- path: test_preparation/test_list.txt
- - store_artifacts:
- path: test_preparation/doctest_list.txt
- - store_artifacts:
- path: ~/transformers/test_preparation/filtered_test_list.txt
- - store_artifacts:
- path: test_preparation/examples_test_list.txt
- - run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
- - run: |
- if [ ! -s test_preparation/generated_config.yml ]; then
- echo "No tests to run, exiting early!"
- circleci-agent step halt
- fi
+ path: test_preparation
+
+ - run:
+ name: "Retrieve Artifact Paths"
+ # [reference] https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts
+ # `CIRCLE_TOKEN` is defined as an environment variables set within a context, see `https://circleci.com/docs/contexts/`
+ command: |
+ project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
+ job_number=${CIRCLE_BUILD_NUM}
+ url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
+ curl -o test_preparation/artifacts.json ${url} --header "Circle-Token: $CIRCLE_TOKEN"
+ - run:
+ name: "Prepare pipeline parameters"
+ command: |
+ python utils/process_test_artifacts.py 
+ 
+ # To avoid too long generated_config.yaml on the continuation orb, we pass the links to the artifacts as parameters.
+ # Otherwise the list of tests was just too big. Explicit is good but for that it was a limitation.
+ # We used:
+
+ # https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts : to get the job artifacts
+ # We could not pass a nested dict, which is why we create the test_file_... parameters for every single job
+
  - store_artifacts:
- path: test_preparation/generated_config.yml
+ path: test_preparation/transformed_artifacts.json
  - store_artifacts:
- path: test_preparation/filtered_test_list_cross_tests.txt
+ path: test_preparation/artifacts.json
  - continuation/continue:
+ parameters: test_preparation/transformed_artifacts.json
  configuration_path: test_preparation/generated_config.yml
 
  # To run all tests for the nightly build
@@ -102,22 +82,49 @@ jobs:
  parallelism: 1
  steps:
  - checkout
- - run: uv pip install -e .
- - run: |
-  mkdir test_preparation
-  echo -n "tests" > test_preparation/test_list.txt
-  echo -n "all" > test_preparation/examples_test_list.txt
-  echo -n "tests/repo_utils" > test_preparation/test_repo_utils.txt
+ - run: uv pip install -U -e .
+ - run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
+ - run: mkdir -p test_preparation
+ - run: python utils/tests_fetcher.py --fetch_all | tee tests_fetched_summary.txt
+ - run: python utils/tests_fetcher.py --filter_tests
+ - run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
  - run: |
- echo -n "tests" > test_list.txt
- python utils/tests_fetcher.py --filter_tests
- mv test_list.txt test_preparation/filtered_test_list.txt
- - run: python .circleci/create_circleci_config.py --fetcher_folder test_preparation
- - run: cp test_preparation/generated_config.yml test_preparation/generated_config.txt
+ if [ ! -s test_preparation/generated_config.yml ]; then
+ echo "No tests to run, exiting early!"
+ circleci-agent step halt
+ fi
+
+ - store_artifacts:
+ path: test_preparation
+
+ - run:
+ name: "Retrieve Artifact Paths"
+ env:
+ CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
+ command: |
+ project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
+ job_number=${CIRCLE_BUILD_NUM}
+ url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
+ curl -o test_preparation/artifacts.json ${url}
+ - run:
+ name: "Prepare pipeline parameters"
+ command: |
+ python utils/process_test_artifacts.py 
+
+ # To avoid too long generated_config.yaml on the continuation orb, we pass the links to the artifacts as parameters.
+ # Otherwise the list of tests was just too big. Explicit is good but for that it was a limitation.
+ # We used:
+
+ # https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts : to get the job artifacts
+ # We could not pass a nested dict, which is why we create the test_file_... parameters for every single job
+
  - store_artifacts:
- path: test_preparation/generated_config.txt
+ path: test_preparation/transformed_artifacts.json
+ - store_artifacts:
+ path: test_preparation/artifacts.json
  - continuation/continue:
- configuration_path: test_preparation/generated_config.yml
+ parameters: test_preparation/transformed_artifacts.json
+ configuration_path: test_preparation/generated_config.yml
 
  check_code_quality:
  working_directory: ~/transformers
@@ -130,7 +137,7 @@ jobs:
  parallelism: 1
  steps:
  - checkout
- - run: uv pip install -e .
+ - run: uv pip install -e ".[quality]"
  - run:
  name: Show installed libraries and their versions
  command: pip freeze | tee installed.txt
@@ -155,13 +162,14 @@ jobs:
  parallelism: 1
  steps:
  - checkout
- - run: uv pip install -e .
+ - run: uv pip install -e ".[quality]"
  - run:
  name: Show installed libraries and their versions
  command: pip freeze | tee installed.txt
  - store_artifacts:
  path: ~/transformers/installed.txt
  - run: python utils/check_copies.py
+ - run: python utils/check_modular_conversion.py
  - run: python utils/check_table.py
  - run: python utils/check_dummies.py
  - run: python utils/check_repo.py
@@ -183,7 +191,10 @@ workflows:
  - check_circleci_user
  - check_code_quality
  - check_repository_consistency
- - fetch_tests
+ - fetch_tests:
+ # [reference] https://circleci.com/docs/contexts/
+ context:
+ - TRANSFORMERS_CONTEXT
 
  nightly:
  when: <<pipeline.parameters.nightly>>