From c5c50834a05e6cebaf042f0f8e1e755804404d73 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Wed, 28 Oct 2020 17:59:43 +0000 Subject: [PATCH] Update CI cache (#8126) --- .circleci/config.yml | 68 ++++++++++++++-------------- .github/workflows/self-push.yml | 12 ++--- .github/workflows/self-scheduled.yml | 12 ++--- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d101673cf7e79..75be5f9e7e86d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,13 +74,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-torch_and_tf-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-torch_and_tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,tf-cpu,torch,testing] - save_cache: - key: v0.3-{{ checksum "setup.py" }} + key: v0.4-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pt_tf_cross_test --durations=0 | tee output.txt @@ -99,13 +99,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-torch-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-torch-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,torch,testing] - save_cache: - key: v0.3-torch-{{ checksum "setup.py" }} + key: v0.4-torch-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=tests ./tests/ | tee tests_output.txt @@ -113,7 +113,7 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports - + run_tests_tf: working_directory: ~/transformers docker: @@ -126,13 +126,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-tf-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,tf-cpu,testing] - save_cache: - key: v0.3-tf-{{ checksum "setup.py" }} + key: v0.4-tf-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt @@ -151,13 +151,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-flax-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-flax-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: sudo pip install .[flax,sklearn,torch,testing] - save_cache: - key: v0.3-flax-{{ checksum "setup.py" }} + key: v0.4-flax-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt @@ -176,13 +176,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-torch-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-torch-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,torch,testing] - save_cache: - key: v0.3-torch-{{ checksum "setup.py" }} + key: v0.4-torch-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt @@ -201,13 +201,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-tf-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,tf-cpu,testing] - save_cache: - key: v0.3-tf-{{ checksum "setup.py" }} + key: v0.4-tf-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt @@ -224,13 +224,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-custom_tokenizers-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-custom_tokenizers-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[ja,testing] - run: python -m unidic download - save_cache: - key: v0.3-custom_tokenizers-{{ checksum "setup.py" }} + key: v0.4-custom_tokenizers-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt @@ -249,13 +249,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-torch_examples-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-torch_examples-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing] - run: pip install -r examples/requirements.txt - save_cache: - key: v0.3-torch_examples-{{ checksum "setup.py" }} + key: v0.4-torch_examples-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=examples ./examples/ | tee examples_output.txt @@ -271,12 +271,12 @@ jobs: - checkout - restore_cache: keys: - - v0.3-build_doc-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-build_doc-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[tf,torch,sentencepiece,docs] - save_cache: - key: v0.3-build_doc-{{ checksum "setup.py" }} + key: v0.4-build_doc-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: cd docs && make html SPHINXOPTS="-W" @@ -293,11 +293,11 @@ jobs: - checkout - restore_cache: keys: - - v0.3-deploy_doc-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-deploy_doc-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install .[tf,torch,sentencepiece,docs] - save_cache: - key: v0.3-deploy_doc-{{ checksum "setup.py" }} + key: v0.4-deploy_doc-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: ./.circleci/deploy.sh @@ -311,13 +311,13 @@ jobs: - checkout - restore_cache: keys: - - v0.3-code_quality-{{ checksum "setup.py" }} - - v0.3-{{ checksum "setup.py" }} + - v0.4-code_quality-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install isort - run: pip install .[tf,torch,flax,quality] - save_cache: - key: v0.3-code_quality-{{ checksum "setup.py" }} + key: v0.4-code_quality-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - run: black --check examples templates tests src utils diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 8a9a25bc0aa155..5836b1034407a4 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -1,10 +1,10 @@ name: Self-hosted runner (push) -on: +on: push: branches: - master - paths: + paths: - "src/**" - "tests/**" - ".github/**" @@ -22,7 +22,7 @@ jobs: which python python --version pip --version - + - name: Current dir run: pwd - run: nvidia-smi @@ -32,7 +32,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + key: v1-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -41,7 +41,7 @@ jobs: which python python --version pip --version - + - name: Install dependencies run: | source .env/bin/activate @@ -84,7 +84,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} + key: v1-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index d480b25964d130..cd9d782b64519d 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -19,18 +19,18 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + key: v1-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | which python python --version pip --version - + - name: Current dir run: pwd - run: nvidia-smi - + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) if: steps.cache.outputs.cache-hit != 'true' run: | @@ -39,7 +39,7 @@ jobs: which python python --version pip --version - + - name: Install dependencies run: | source .env/bin/activate @@ -81,7 +81,7 @@ jobs: RUN_PIPELINE_TESTS: yes run: | source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests + python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests - name: test suite reports artifacts if: ${{ always() }} @@ -101,7 +101,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} + key: v1-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} - name: Python version run: |