Add IQ3_ML, reinstate IQ3_XXXL

Author: Nexesenex

examples/quantize/quantize.cpp

@@ -37,8 +37,10 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
     { "IQ3_XXS",  LLAMA_FTYPE_MOSTLY_IQ3_XXS,  " 3.06 bpw quantization",            },
     { "IQ3_S",    LLAMA_FTYPE_MOSTLY_IQ3_S,    " 3.44 bpw quantization",            },
     { "IQ3_M",    LLAMA_FTYPE_MOSTLY_IQ3_M,    " 3.70 bpw quantization mix",        },
+    { "IQ3_ML",   LLAMA_FTYPE_MOSTLY_IQ3_ML,   " 3.80 bpw quantization mix",        },
     { "IQ3_XL",   LLAMA_FTYPE_MOSTLY_IQ3_XL,   " 3.90 bpw quantization mix",        },
-    { "IQ3_XXL",  LLAMA_FTYPE_MOSTLY_IQ3_XXL,  " 4.10 bpw quantization mix",        },
+    { "IQ3_XXL",  LLAMA_FTYPE_MOSTLY_IQ3_XXL,  " 4.00 bpw quantization mix",        },
+    { "IQ3_XXXL", LLAMA_FTYPE_MOSTLY_IQ3_XXXL, " 4.10 bpw quantization mix",        },
     { "Q3_K",     LLAMA_FTYPE_MOSTLY_Q3_K_M,   "alias for Q3_K_M"                   },
     { "IQ3_XS",   LLAMA_FTYPE_MOSTLY_IQ3_XS,   " 3.3 bpw quantization",             },
     { "Q3_K_S",   LLAMA_FTYPE_MOSTLY_Q3_K_S,   " 3.41G, +1.6321 ppl @ Llama-3-8B",  },

gguf-py/gguf/constants.py

@@ -1370,7 +1370,7 @@ class LlamaFileType(IntEnum):
     MOSTLY_Q2_K          = 10  # except 1d tensors
     MOSTLY_Q3_K_S        = 11  # except 1d tensors
     MOSTLY_Q3_K_M        = 12  # except 1d tensors
-    MOSTLY_Q3_K_XL       = 13  # except 1d tensors
+    MOSTLY_Q3_K_L        = 13  # except 1d tensors
     MOSTLY_Q4_K_S        = 14  # except 1d tensors
     MOSTLY_Q4_K_M        = 15  # except 1d tensors
     MOSTLY_Q5_K_S        = 16  # except 1d tensors
@@ -1402,7 +1402,9 @@ class LlamaFileType(IntEnum):
     MOSTLY_IQ1_XL        = 42  # except 1d tensors
     MOSTLY_IQ4_XSR       = 43  # except 1d tensors
     MOSTLY_IQ3_XXL       = 44  # except 1d tensors
-    MOSTLY_Q3_K_L        = 45  # except 1d tensors
+    MOSTLY_Q3_K_XL       = 45  # except 1d tensors
+    MOSTLY_IQ3_ML        = 46  # except 1d tensors
+    MOSTLY_IQ3_XXXL      = 47  # except 1d tensors
 
     GUESSED              = 1024  # not specified in the model file
 

include/llama.h

@@ -149,7 +149,7 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_Q2_K          = 10, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q3_K_S        = 11, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q3_K_M        = 12, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_XL       = 13, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q3_K_L        = 13, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_K_S        = 14, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_K_M        = 15, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q5_K_S        = 16, // except 1d tensors
@@ -181,7 +181,10 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_IQ1_XL        = 42, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_IQ4_XSR       = 43, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_IQ3_XXL       = 44, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_L        = 45, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q3_K_XL       = 45, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_ML        = 46, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_XXXL      = 47, // except 1d tensors
+        LLAMA_FTYPE_CQS                  = 99, // except 1d tensors
 
         LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
     };