merge

Author: ebsmothers

.github/workflows/build_linux_wheels.yaml

@@ -27,37 +27,8 @@ jobs:
       with-cuda: enable
       with-rocm: enable
       build-python-only: enable
-  # TODO: Remove `filter-python-version` after PyArrow releases v18
-  filter-python-versions:
-    needs: generate-matrix
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Filter matrix to exclude Python 3.13
-        id: set-matrix
-        shell: python
-        env:
-          input-matrix: ${{ needs.generate-matrix.outputs.matrix }}
-        run: |
-          import os
-          import json
-
-          # Grab environment variables
-          input_matrix = json.loads(os.environ["input-matrix"])
-          github_output_file = os.environ["GITHUB_OUTPUT"]
-
-          # Filter out any builds for 3.13
-          filtered_matrix = {"include": []}
-          for build in input_matrix["include"]:
-            if build["python_version"] != "3.13":
-              filtered_matrix["include"].append(build)
-
-          # Write the new matrix to the default outputs file
-          with open(github_output_file, "w") as handle:
-            handle.write(f"matrix={json.dumps(filtered_matrix)}")
   build:
-    needs: filter-python-versions
+    needs: generate-matrix
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     strategy:
@@ -66,7 +37,7 @@ jobs:
       repository: pytorch/torchtune
       ref: ""
       package-name: torchtune
-      build-matrix: ${{ needs.filter-python-versions.outputs.matrix }}
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
       pre-script: .github/scripts/pre_build_script.sh
       trigger-event: ${{ github.event_name }}
       build-platform: 'python-build-package'

.github/workflows/regression_test.yaml

@@ -26,6 +26,8 @@ jobs:
         python-version: ['3.11']
         torch-version: ["stable", "nightly"]
       fail-fast: false
+    env:
+      PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
     steps:
       - name: Check out repo
         uses: actions/checkout@v3

README.md

@@ -9,8 +9,11 @@
 
 [**Introduction**](#introduction) | [**Installation**](#installation) | [**Get Started**](#get-started) |  [**Documentation**](https://pytorch.org/torchtune/main/index.html) | [**Community**](#community) | [**License**](#license) | [**Citing torchtune**](#citing-torchtune)
 
-> [!IMPORTANT]
-> Update September 25, 2024: torchtune has support for **Llama 3.2 11B Vision**, **Llama 3.2 3B**, and **Llama 3.2 1B** models! Try them out by following our installation instructions [here](#Installation), then run any of the text configs [here](recipes/configs/llama3_2) or vision configs [here](recipes/configs/llama3_2_vision).
+### 📣 Recent updates 📣
+* *November 2024*: torchtune has released [v0.4.0](https://github.com/pytorch/torchtune/releases/tag/v0.4.0) which includes stable support for exciting features like activation offloading and multimodal QLoRA
+* *November 2024*: torchtune has added [Gemma2](recipes/configs/gemma2) to its models!
+* *October 2024*: torchtune added support for Qwen2.5 models - find the recipes [here](recipes/configs/qwen2_5/)
+* *September 2024*: torchtune has support for **Llama 3.2 11B Vision**, **Llama 3.2 3B**, and **Llama 3.2 1B** models! Try them out by following our installation instructions [here](#Installation), then run any of the text configs [here](recipes/configs/llama3_2) or vision configs [here](recipes/configs/llama3_2_vision).
 
 
  
@@ -44,8 +47,10 @@ torchtune currently supports the following models.
 | [Code-Llama2](https://ai.meta.com/blog/code-llama-large-language-model-coding/)   | 7B, 13B, 70B [[models](torchtune/models/code_llama2/_model_builders.py), [configs](recipes/configs/code_llama2/)] |
 | [Mistral](https://huggingface.co/mistralai)   | 7B [[models](torchtune/models/mistral/_model_builders.py), [configs](recipes/configs/mistral/)] |
 | [Gemma](https://huggingface.co/collections/google/gemma-release-65d5efbccdbb8c4202ec078b)   | 2B, 7B [[models](torchtune/models/gemma/_model_builders.py), [configs](recipes/configs/gemma/)] |
+| [Gemma2](https://huggingface.co/docs/transformers/main/en/model_doc/gemma2)   | 2B, 9B, 27B [[models](torchtune/models/gemma2/_model_builders.py), [configs](recipes/configs/gemma2/)] |
 | [Microsoft Phi3](https://huggingface.co/collections/microsoft/phi-3-6626e15e9585a200d2d761e3) | Mini [[models](torchtune/models/phi3/), [configs](recipes/configs/phi3/)]
 | [Qwen2](https://qwenlm.github.io/blog/qwen2/) | 0.5B, 1.5B, 7B [[models](torchtune/models/qwen2/), [configs](recipes/configs/qwen2/)]
+| [Qwen2.5](https://qwenlm.github.io/blog/qwen2.5/) | 0.5B, 1.5B, 3B, 7B, 14B, 32B, 72B [[models](torchtune/models/qwen2_5/), [configs](recipes/configs/qwen2_5/)]
 
 We're always adding new models, but feel free to [file an issue](https://github.com/pytorch/torchtune/issues/new) if there's a new one you would like to see in torchtune.
 
@@ -162,6 +167,7 @@ To download Llama3.1, you can run:
 ```bash
 tune download meta-llama/Meta-Llama-3.1-8B-Instruct \
 --output-dir /tmp/Meta-Llama-3.1-8B-Instruct \
+--ignore-patterns "original/consolidated.00.pth" \
 --hf-token <HF_TOKEN> \
 ```
 
@@ -258,6 +264,7 @@ We really value our community and the contributions made by our wonderful users.
 - [@fyabc](https://github.com/fyabc) for adding Qwen2 models, tokenizer, and recipe integration to torchtune
 - [@solitude-alive](https://github.com/solitude-alive) for adding the [Gemma 2B model](torchtune/models/gemma/) to torchtune, including recipe changes, numeric validations of the models and recipe correctness
 - [@yechenzhi](https://github.com/yechenzhi) for adding [Direct Preference Optimization (DPO)](recipes/lora_dpo_single_device.py) to torchtune, including the recipe and config along with correctness checks
+- [@Optimox](https://github.com/Optimox) for adding all the [Gemma2 variants](torchtune/models/gemma2) to torchtune!
 
 
 &nbsp;

docs/source/_templates/layout.html

@@ -15,7 +15,13 @@
     var collapsedSections = ['Introduction', 'Getting Started', 'Tutorials']
 </script> -->
 
-<script script type="text/javascript">
+<script type="text/javascript">
     var collapsedSections = []
 </script>
+{{ super() }}
+<script type="text/javascript">
+    $(document).ready(function() {
+        $(".main-menu-item a[href='https://github.com/pytorch/pytorch']").attr("href", "https://github.com/pytorch/torchtune");
+    });
+</script>
 {% endblock %}

docs/source/api_ref_data.rst

@@ -6,8 +6,6 @@ torchtune.data
 
 .. currentmodule:: torchtune.data
 
-.. _chat_formats:
-
 Text templates
 --------------
 
@@ -18,14 +16,12 @@ and models.
     :toctree: generated/
     :nosignatures:
 
-    InstructTemplate
     GrammarErrorCorrectionTemplate
     SummarizeTemplate
     QuestionAnswerTemplate
     PromptTemplate
     PromptTemplateInterface
     ChatMLTemplate
-    ChatFormat
 
 Types
 -----
@@ -37,18 +33,6 @@ Types
     Message
     Role
 
-Converters
-----------
-
-Converts data from common JSON formats into a torchtune :class:`Message`.
-
-.. autosummary::
-    :toctree: generated/
-    :nosignatures:
-
-    get_sharegpt_messages
-    get_openai_messages
-
 .. _message_transforms_ref:
 
 Message transforms

docs/source/api_ref_datasets.rst

@@ -6,11 +6,11 @@ torchtune.datasets
 
 .. currentmodule:: torchtune.datasets
 
-For a detailed general usage guide, please see our :ref:`datasets tutorial <dataset_tutorial_label>`.
+For a detailed general usage guide, please see :ref:`datasets_overview`.
 
 
 Text datasets
-------------------
+-------------
 
 torchtune supports several widely used text-only datasets to help quickly bootstrap your fine-tuning.
 
@@ -37,6 +37,7 @@ Image + Text datasets
 
     multimodal.llava_instruct_dataset
     multimodal.the_cauldron_dataset
+    multimodal.vqa_dataset
 
 .. _dataset_builders:
 

docs/source/api_ref_models.rst

@@ -208,6 +208,47 @@ To download the CodeLlama-7B model:
     code_llama2.lora_code_llama2_70b
     code_llama2.qlora_code_llama2_70b
 
+qwen-2.5
+--------
+
+Models of size 0.5B, 1.5B, 3B, 7B, 14B, 32B, 72B from the `Qwen2.5 family <https://huggingface.co/collections/Qwen/qwen25-66e81a666513e518adb90d9e>`_.
+
+To download the Qwen2.5 1.5B model, for example:
+
+.. code-block:: bash
+
+    tune download Qwen/Qwen2.5-1.5B-Instruct --output-dir /tmp/Qwen2_5-1_5B-Instruct --ignore-patterns None
+
+.. autosummary::
+    :toctree: generated/
+    :nosignatures:
+
+    qwen2_5.qwen2_5_0_5b
+    qwen2_5.lora_qwen2_5_0_5b
+    qwen2_5.qwen2_5_1_5b_base
+    qwen2_5.qwen2_5_1_5b_instruct
+    qwen2_5.lora_qwen2_5_1_5b_base
+    qwen2_5.lora_qwen2_5_1_5b_instruct
+    qwen2_5.qwen2_5_3b
+    qwen2_5.lora_qwen2_5_3b
+    qwen2_5.qwen2_5_7b_base
+    qwen2_5.qwen2_5_7b_instruct
+    qwen2_5.lora_qwen2_5_7b_base
+    qwen2_5.lora_qwen2_5_7b_instruct
+    qwen2_5.qwen2_5_14b_base
+    qwen2_5.qwen2_5_14b_instruct
+    qwen2_5.lora_qwen2_5_14b_base
+    qwen2_5.lora_qwen2_5_14b_instruct
+    qwen2_5.qwen2_5_32b_base
+    qwen2_5.qwen2_5_32b_instruct
+    qwen2_5.lora_qwen2_5_32b_base
+    qwen2_5.lora_qwen2_5_32b_instruct
+    qwen2_5.qwen2_5_72b_base
+    qwen2_5.qwen2_5_72b_instruct
+    qwen2_5.lora_qwen2_5_72b_base
+    qwen2_5.lora_qwen2_5_72b_instruct
+    qwen2_5.qwen2_5_tokenizer
+
 qwen-2
 ------
 
@@ -225,12 +266,12 @@ To download the Qwen2 1.5B model, for example:
 
     qwen2.qwen2
     qwen2.lora_qwen2
-    qwen2.qwen2_7b
     qwen2.qwen2_0_5b
-    qwen2.qwen2_1_5b
-    qwen2.lora_qwen2_7b
     qwen2.lora_qwen2_0_5b
+    qwen2.qwen2_1_5b
     qwen2.lora_qwen2_1_5b
+    qwen2.qwen2_7b
+    qwen2.lora_qwen2_7b
     qwen2.qwen2_tokenizer
 
 phi-3
@@ -320,8 +361,39 @@ To download the Gemma 7B model:
     gemma.gemma_tokenizer
 
 
+gemma2 :
+--------
+
+Models of size 2B, 9B, 27B from the `Gemma family <https://blog.google/technology/developers/gemma-open-models/>`_.
+
+Important: You need to request access on `Hugging Face <https://huggingface.co/google/gemma-2-2b>`__ to use this model.
+
+To download the Gemma2 2B, 9B, 27B models :
+
+.. code-block:: bash
+
+    tune download google/gemma-2-<MODEL_SIZE>b --ignore-patterns "gemma-2-<MODEL_SIZE>b.gguf"  --hf-token <HF_TOKEN>
+
+
+.. autosummary::
+    :toctree: generated/
+    :nosignatures:
+
+    gemma2.gemma2
+    gemma2.lora_gemma2
+    gemma2.gemma2_2b
+    gemma2.lora_gemma2_2b
+    gemma2.qlora_gemma2_2b
+    gemma2.gemma2_9b
+    gemma2.lora_gemma2_9b
+    gemma2.qlora_gemma2_9b
+    gemma2.gemma2_27b
+    gemma2.lora_gemma2_27b
+    gemma2.qlora_gemma2_27b
+    gemma.gemma_tokenizer
+
 clip
------
+----
 
 Vision components to support multimodality using `CLIP encoder <https://arxiv.org/abs/2103.00020>`_.
 

docs/source/api_ref_modules.rst

@@ -71,9 +71,11 @@ PEFT Components
     :nosignatures:
 
     peft.LoRALinear
+    peft.DoRALinear
     peft.AdapterModule
     peft.get_adapter_params
     peft.set_trainable_params
+    peft.get_adapter_state_dict
     peft.validate_missing_and_unexpected_for_lora
     peft.disable_adapter
 

docs/source/api_ref_training.rst

@@ -53,6 +53,7 @@ Utilities for enabling and working with distributed training.
     init_distributed
     is_distributed
     get_world_size_and_rank
+    gather_cpu_state_dict
 
 .. _ac_label: