Merge pull request #85 from calpt/sync/v3.5.1

adapter-hub · Nov 19, 2020 · 0270fdd · 0270fdd
2 parents cee459e + 659285c
commit 0270fdd
Show file tree

Hide file tree

Showing 804 changed files with 65,837 additions and 20,918 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -74,21 +74,20 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-torch_and_tf-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-torch_and_tf-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install git+https://github.com/huggingface/datasets
  - run: pip install .[sklearn,tf-cpu,torch,testing]
- - run: pip install codecov pytest-cov
  - save_cache:
- key: v0.3-{{ checksum "setup.py" }}
+ key: v0.4-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ --cov | tee output.txt
- - run: codecov
+ - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf ./tests/ -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
  - store_artifacts:
- path: ~/transformers/output.txt
- destination: test_output.txt
+ path: ~/transformers/tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/reports
+
  run_tests_torch:
  working_directory: ~/transformers
  docker:
@@ -101,19 +100,20 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-torch-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-torch-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install git+https://github.com/huggingface/datasets
  - run: pip install .[sklearn,torch,testing]
  - save_cache:
- key: v0.3-torch-{{ checksum "setup.py" }}
+ key: v0.4-torch-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
+ - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
  - store_artifacts:
- path: ~/transformers/output.txt
- destination: test_output.txt
+ path: ~/transformers/reports
+
  run_tests_tf:
  working_directory: ~/transformers
  docker:
@@ -126,19 +126,98 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-tf-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-tf-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
+ - run: pip install --upgrade pip
+ - run: pip install .[sklearn,tf-cpu,testing]
+ - save_cache:
+ key: v0.4-tf-{{ checksum "setup.py" }}
+ paths:
+ - '~/.cache/pip'
+ - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf ./tests/ | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/reports
+
+ run_tests_flax:
+ working_directory: ~/transformers
+ docker:
+ - image: circleci/python:3.7
+ environment:
+ OMP_NUM_THREADS: 1
+ resource_class: xlarge
+ parallelism: 1
+ steps:
+ - checkout
+ - restore_cache:
+ keys:
+ - v0.4-flax-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
+ - run: pip install --upgrade pip
+ - run: sudo pip install .[flax,sklearn,torch,testing]
+ - save_cache:
+ key: v0.4-flax-{{ checksum "setup.py" }}
+ paths:
+ - '~/.cache/pip'
+ - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax ./tests/ | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/reports
+
+ run_tests_pipelines_torch:
+ working_directory: ~/transformers
+ docker:
+ - image: circleci/python:3.7
+ environment:
+ OMP_NUM_THREADS: 1
+ resource_class: xlarge
+ parallelism: 1
+ steps:
+ - checkout
+ - restore_cache:
+ keys:
+ - v0.4-torch-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
+ - run: pip install --upgrade pip
+ - run: pip install .[sklearn,torch,testing]
+ - save_cache:
+ key: v0.4-torch-{{ checksum "setup.py" }}
+ paths:
+ - '~/.cache/pip'
+ - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test ./tests/ | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/reports
+
+ run_tests_pipelines_tf:
+ working_directory: ~/transformers
+ docker:
+ - image: circleci/python:3.7
+ environment:
+ OMP_NUM_THREADS: 1
+ resource_class: xlarge
+ parallelism: 1
+ steps:
+ - checkout
+ - restore_cache:
+ keys:
+ - v0.4-tf-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install git+https://github.com/huggingface/datasets
  - run: pip install .[sklearn,tf-cpu,testing]
  - save_cache:
- key: v0.3-tf-{{ checksum "setup.py" }}
+ key: v0.4-tf-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
+ - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf ./tests/ -m is_pipeline_test | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
  - store_artifacts:
- path: ~/transformers/output.txt
- destination: test_output.txt
+  path: ~/transformers/reports
+
  run_tests_custom_tokenizers:
  working_directory: ~/transformers
  docker:
@@ -149,19 +228,21 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-custom_tokenizers-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
  - run: pip install .[ja,testing]
  - run: python -m unidic download
  - save_cache:
- key: v0.3-custom_tokenizers-{{ checksum "setup.py" }}
+ key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
+ - run: python -m pytest -s --make-reports=tests_custom_tokenizers ./tests/test_tokenization_bert_japanese.py | tee tests_output.txt
+ - store_artifacts:
+ path: ~/transformers/tests_output.txt
  - store_artifacts:
- path: ~/transformers/output.txt
- destination: test_output.txt
+  path: ~/transformers/reports
+
  run_examples_torch:
  working_directory: ~/transformers
  docker:
@@ -174,19 +255,21 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-torch_examples-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-torch_examples-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
  - run: pip install .[sklearn,torch,testing]
  - run: pip install -r examples/requirements.txt
  - save_cache:
- key: v0.3-torch_examples-{{ checksum "setup.py" }}
+ key: v0.4-torch_examples-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: python -m pytest -n 8 --dist=loadfile -rA -s ./examples/ | tee output.txt
+ - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/ | tee examples_output.txt
+ - store_artifacts:
+ path: ~/transformers/examples_output.txt
  - store_artifacts:
- path: ~/transformers/output.txt
- destination: test_output.txt
+ path: ~/transformers/reports
+
  build_doc:
  working_directory: ~/transformers
  docker:
@@ -195,17 +278,18 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-build_doc-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-build_doc-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install .[tf,torch,docs]
+ - run: pip install .[tf,torch,sentencepiece,docs]
  - save_cache:
- key: v0.3-build_doc-{{ checksum "setup.py" }}
+ key: v0.4-build_doc-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
  - run: cd docs && make html SPHINXOPTS="-W"
  - store_artifacts:
  path: ./docs/_build
+
  deploy_doc:
  working_directory: ~/transformers
  docker:
@@ -217,14 +301,15 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-deploy_doc-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
- - run: pip install .[tf,torch,docs]
+ - v0.4-deploy_doc-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
+ - run: pip install .[tf,torch,sentencepiece,docs]
  - save_cache:
- key: v0.3-deploy_doc-{{ checksum "setup.py" }}
+ key: v0.4-deploy_doc-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
  - run: ./.circleci/deploy.sh
+
  check_code_quality:
  working_directory: ~/transformers
  docker:
@@ -235,20 +320,23 @@ jobs:
  - checkout
  - restore_cache:
  keys:
- - v0.3-code_quality-{{ checksum "setup.py" }}
- - v0.3-{{ checksum "setup.py" }}
+ - v0.4-code_quality-{{ checksum "setup.py" }}
+ - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
  - run: pip install isort
- - run: pip install .[tf,torch,quality]
+ - run: pip install .[tf,torch,flax,quality]
  - save_cache:
- key: v0.3-code_quality-{{ checksum "setup.py" }}
+ key: v0.4-code_quality-{{ checksum "setup.py" }}
  paths:
  - '~/.cache/pip'
- - run: black --check examples templates tests src utils
- - run: isort --check-only examples templates tests src utils
- - run: flake8 examples templates tests src utils
+ - run: black --check examples tests src utils
+ - run: isort --check-only examples tests src utils
+ - run: flake8 examples tests src utils
+ - run: python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
  - run: python utils/check_copies.py
+ - run: python utils/check_dummies.py
  - run: python utils/check_repo.py
+
  check_repository_consistency:
  working_directory: ~/transformers
  docker:
@@ -279,6 +367,7 @@ jobs:
  - setup_remote_docker
  - *build_push_docker
  - *deploy_cluster
+
  cleanup-gke-jobs:
  docker:
  - image: circleci/python:3.6
@@ -288,6 +377,7 @@ jobs:
  cluster: $GKE_CLUSTER
  perform-login: true
  - *delete_gke_jobs
+
 workflow_filters: &workflow_filters
  filters:
  branches:
@@ -304,6 +394,9 @@ workflows:
  - run_tests_torch_and_tf
  - run_tests_torch
  - run_tests_tf
+ - run_tests_flax
+ - run_tests_pipelines_torch
+ - run_tests_pipelines_tf
  - build_doc
  - deploy_doc: *workflow_filters
  tpu_testing_jobs:

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
@@ -50,4 +50,5 @@ deploy_doc "b42586e" v2.11.0
 deploy_doc "7fb8bdf" v3.0.2
 deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
-deploy_doc "0613f05" # v3.3.0 Latest stable release
+deploy_doc "0613f05" v3.3.1
+deploy_doc "eb0e0ce" # v3.4.0 Latest stable release
diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md
@@ -30,20 +30,22 @@ assignees: ''
  Trainer: @sgugger
  Speed and Memory Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
- Translation: @sshleifer
- Summarization: @sshleifer
  TextGeneration: @TevenLeScao 
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @TevenLeScao
- blenderbot: @mariamabarham
- Bart: @sshleifer
- Marian: @sshleifer
+ Text Generation: @patrickvonplaten @TevenLeScao
+ Blenderbot: @patrickvonplaten
+ Bart: @patrickvonplaten
+ Marian: @patrickvonplaten
+ Pegasus: @patrickvonplaten
+ mBART: @patrickvonplaten
  T5: @patrickvonplaten
  Longformer/Reformer: @patrickvonplaten
- TransfoXL/XLNet: @TevenLeScao 
- examples/seq2seq: @sshleifer
+ TransfoXL/XLNet: @TevenLeScao
+ RAG: @patrickvonplaten, @lhoestq
+ FSMT: @stas00
+ examples/seq2seq: @patil-suraj
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it

diff --git a/.gitignore b/.gitignore
@@ -9,9 +9,11 @@ __pycache__/
 *.so
 
 # tests and logs
-tests/fixtures
+tests/fixtures/*
+!tests/fixtures/sample_text_no_unicode.txt
 logs/
 lightning_logs/
+lang_code_data/
 
 # Distribution / packaging
 .Python
@@ -157,5 +159,7 @@ debug.env
 #ctags
 tags
 
+# pre-commit
+.pre-commit*
 # scripts
 *.ps1
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -96,7 +96,7 @@ folder.
 
 ## Start contributing! (Pull Requests)
 
-Before writing code, we strongly advise you to search through the exising PRs or
+Before writing code, we strongly advise you to search through the existing PRs or
 issues to make sure that nobody is already working on the same thing. If you are
 unsure, it is always a good idea to open an issue to get some feedback.
 
@@ -235,7 +235,7 @@ Follow these steps to start contributing:
 ### Checklist
 
 1. The title of your pull request should be a summary of its contribution;
-2. If your pull request adresses an issue, please mention the issue number in
+2. If your pull request addresses an issue, please mention the issue number in
  the pull request description to make sure they are linked (and people
  consulting the issue know you are working on it);
 3. To indicate a work in progress please prefix the title with `[WIP]`. These