Merge pull request #2 from ylacombe/add-musicgen-melody

Add musicgen melody

.circleci/create_circleci_config.py

@@ -52,7 +52,7 @@ class CircleCIJob:
  name: str
  additional_env: Dict[str, Any] = None
  cache_name: str = None
- cache_version: str = "0.8"
+ cache_version: str = "0.8.2"
  docker_image: List[Dict[str, str]] = None
  install_steps: List[str] = None
  marker: Optional[str] = None
@@ -128,22 +128,6 @@ def to_dict(self):
  steps.extend([{"run": l} for l in self.install_steps])
  steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
  steps.extend([{"run": "pip install pytest-subtests"}])
- steps.append(
- {
- "save_cache": {
- "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
- "paths": ["~/.cache/pip"],
- }
- }
- )
- steps.append(
- {
- "save_cache": {
- "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
- "paths": ["~/.pyenv/versions/"],
- }
- }
- )
  steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
  steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
 
@@ -264,6 +248,25 @@ def to_dict(self):
 
  steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}})
  steps.append({"store_artifacts": {"path": "~/transformers/reports"}})
+
+ # save cache at the end: so pytest step runs before cache saving and we can see results earlier
+ steps.append(
+ {
+ "save_cache": {
+ "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
+ "paths": ["~/.cache/pip"],
+ }
+ }
+ )
+ steps.append(
+ {
+ "save_cache": {
+ "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
+ "paths": ["~/.pyenv/versions/"],
+ }
+ }
+ )
+
  job["steps"] = steps
  return job
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -17,7 +17,7 @@ Fixes # (issue)
 
 ## Before submitting
 - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
-- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests),
+- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
  Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
  to it if that's the case.

.github/workflows/build-docker-images.yml

@@ -20,18 +20,8 @@ concurrency:
 jobs:
  latest-docker:
  name: "Latest PyTorch + TensorFlow [dev]"
- runs-on: ubuntu-22.04
+ runs-on: [intel-cpu, 8-cpu, ci]
  steps:
- - name: Cleanup disk
- run: |
- sudo ls -l /usr/local/lib/
- sudo ls -l /usr/share/
- sudo du -sh /usr/local/lib/
- sudo du -sh /usr/share/
- sudo rm -rf /usr/local/lib/android
- sudo rm -rf /usr/share/dotnet
- sudo du -sh /usr/local/lib/
- sudo du -sh /usr/share/
  -
  name: Set up Docker Buildx
  uses: docker/setup-buildx-action@v3
@@ -307,3 +297,31 @@ jobs:
  # REF=main
  # push: true
  # tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci
+
+ latest-quantization-torch-docker:
+ name: "Latest Pytorch + Quantization [dev]"
+ # Push CI doesn't need this image
+ if: inputs.image_postfix != '-push-ci'
+ runs-on: [intel-cpu, 8-cpu, ci]
+ steps:
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ -
+ name: Check out code
+ uses: actions/checkout@v3
+ -
+ name: Login to DockerHub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_PASSWORD }}
+ -
+ name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ context: ./docker/transformers-quantization-latest-gpu
+ build-args: |
+ REF=main
+ push: true
+ tags: huggingface/transformers-quantization-latest-gpu${{ inputs.image_postfix }}

.github/workflows/self-scheduled.yml

@@ -265,7 +265,7 @@ jobs:
  working-directory: /workspace
  run: |
  python3 -m pip uninstall -y deepspeed
- DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check
+ DS_DISABLE_NINJA=1 DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check
 
  - name: NVIDIA-SMI
  run: |
@@ -297,6 +297,56 @@ jobs:
  name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports
  path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu
 
+ run_tests_quantization_torch_gpu:
+ name: Quantization tests
+ strategy:
+ fail-fast: false
+ matrix:
+ machine_type: [single-gpu, multi-gpu]
+ runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, daily-ci]
+ container:
+ image: huggingface/transformers-quantization-latest-gpu
+ options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+ needs: setup
+ steps:
+ - name: Update clone
+ working-directory: /transformers
+ run: git fetch && git checkout ${{ github.sha }}
+
+ - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
+ working-directory: /transformers
+ run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
+
+ - name: NVIDIA-SMI
+ run: |
+ nvidia-smi
+
+ - name: Environment
+ working-directory: /transformers
+ run: |
+ python3 utils/print_env.py
+
+ - name: Show installed libraries and their versions
+ working-directory: /transformers
+ run: pip freeze
+
+ - name: Run quantization tests on GPU
+ working-directory: /transformers
+ run: |
+ python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_quantization_torch_gpu tests/quantization
+
+ - name: Failure short reports
+ if: ${{ failure() }}
+ continue-on-error: true
+ run: cat /transformers/reports/${{ matrix.machine_type }}_tests_quantization_torch_gpu/failures_short.txt
+
+ - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_quantization_torch_gpu"
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ matrix.machine_type }}_run_tests_quantization_torch_gpu
+ path: /transformers/reports/${{ matrix.machine_type }}_tests_quantization_torch_gpu
+
  run_extract_warnings:
  name: Extract warnings in CI artifacts
  runs-on: ubuntu-22.04
@@ -307,7 +357,8 @@ jobs:
  run_examples_gpu,
  run_pipelines_tf_gpu,
  run_pipelines_torch_gpu,
- run_all_tests_torch_cuda_extensions_gpu
+ run_all_tests_torch_cuda_extensions_gpu,
+ run_tests_quantization_torch_gpu,
  ]
  steps:
  - name: Checkout transformers
@@ -355,6 +406,7 @@ jobs:
  run_pipelines_tf_gpu,
  run_pipelines_torch_gpu,
  run_all_tests_torch_cuda_extensions_gpu,
+ run_tests_quantization_torch_gpu,
  run_extract_warnings
  ]
  steps:

CONTRIBUTING.md

@@ -102,7 +102,7 @@ We have added [templates](https://github.com/huggingface/transformers/tree/main/
 
 ## Do you want to implement a new model?
 
-New models are constantly released and if you want to implement a new model, please provide the following information
+New models are constantly released and if you want to implement a new model, please provide the following information:
 
 * A short description of the model and a link to the paper.
 * Link to the implementation if it is open-sourced.
@@ -129,7 +129,7 @@ You will need basic `git` proficiency to contribute to
 manual. Type `git --help` in a shell and enjoy! If you prefer books, [Pro
 Git](https://git-scm.com/book/en/v2) is a very good reference.
 
-You'll need **[Python 3.8]((https://github.com/huggingface/transformers/blob/main/setup.py#L426))** or above to contribute to 🤗 Transformers. Follow the steps below to start contributing:
+You'll need **[Python 3.8](https://github.com/huggingface/transformers/blob/main/setup.py#L426)** or above to contribute to 🤗 Transformers. Follow the steps below to start contributing:
 
 1. Fork the [repository](https://github.com/huggingface/transformers) by
  clicking on the **[Fork](https://github.com/huggingface/transformers/fork)** button on the repository's page. This creates a copy of the code
@@ -305,7 +305,7 @@ the [tests](https://github.com/huggingface/transformers/tree/main/tests) folder
 [examples](https://github.com/huggingface/transformers/tree/main/examples) folder.
 
 We like `pytest` and `pytest-xdist` because it's faster. From the root of the
-repository, specify a *path to a subfolder or a test file* to run the test.
+repository, specify a *path to a subfolder or a test file* to run the test:
 
 ```bash
 python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model
@@ -377,7 +377,7 @@ One way to run the `make` command on Windows is with MSYS2:
 3. Run in the shell: `pacman -Syu` and install `make` with `pacman -S make`.
 4. Add `C:\msys64\usr\bin` to your PATH environment variable.
 
-You can now use `make` from any terminal (Powershell, cmd.exe, etc.)! 🎉
+You can now use `make` from any terminal (PowerShell, cmd.exe, etc.)! 🎉
 
 ### Sync a forked repository with upstream main (the Hugging Face repository)
 
@@ -386,9 +386,9 @@ When updating the main branch of a forked repository, please follow these steps
 1. When possible, avoid syncing with the upstream using a branch and PR on the forked repository. Instead, merge directly into the forked main.
 2. If a PR is absolutely necessary, use the following steps after checking out your branch:
 
-```bash
-git checkout -b your-branch-for-syncing
-git pull --squash --no-commit upstream main
-git commit -m '<your message without GitHub references>'
-git push --set-upstream origin your-branch-for-syncing
-```
+ ```bash
+ git checkout -b your-branch-for-syncing
+ git pull --squash --no-commit upstream main
+ git commit -m '<your message without GitHub references>'
+ git push --set-upstream origin your-branch-for-syncing
+ ```