More overhaul for IQ4_XSR and new IQ4_MR

Author: Nexesenex

examples/quantize/quantize.cpp

@@ -51,6 +51,7 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
     { "IQ4_NL",   LLAMA_FTYPE_MOSTLY_IQ4_NL,   " 4.50 bpw non-linear quantization", },
     { "IQ4_XS",   LLAMA_FTYPE_MOSTLY_IQ4_XS,   " 4.25 bpw non-linear quantization", },
     { "IQ4_XSR",  LLAMA_FTYPE_MOSTLY_IQ4_XSR,  " 4.xx bpw non-linear quantization", },
+    { "IQ4_MR",   LLAMA_FTYPE_MOSTLY_IQ4_MR,   " 4.xx bpw non-linear quantization", },
     { "Q4_K",     LLAMA_FTYPE_MOSTLY_Q4_K_M,   "alias for Q4_K_M",                  },
     { "Q4_K_S",   LLAMA_FTYPE_MOSTLY_Q4_K_S,   " 4.37G, +0.2689 ppl @ Llama-3-8B",  },
     { "Q4_K_M",   LLAMA_FTYPE_MOSTLY_Q4_K_M,   " 4.58G, +0.1754 ppl @ Llama-3-8B",  },

gguf-py/gguf/constants.py

@@ -1448,12 +1448,14 @@ class LlamaFileType(IntEnum):
     MOSTLY_Q2_K_L        = 40  # except 1d tensors
     MOSTLY_IQ1_XS        = 41  # except 1d tensors
     MOSTLY_IQ1_XL        = 42  # except 1d tensors
-    MOSTLY_IQ4_XSR       = 43  # except 1d tensors
-    MOSTLY_IQ3_XXL       = 44  # except 1d tensors
-    MOSTLY_Q3_K_XL       = 45  # except 1d tensors
-    MOSTLY_IQ3_ML        = 46  # except 1d tensors
-    MOSTLY_IQ3_XXXL      = 47  # except 1d tensors
-    MOSTLY_IQ3_UXL       = 48  # except 1d tensors
+    MOSTLY_IQ3_XXL       = 43  # except 1d tensors
+    MOSTLY_Q3_K_XL       = 44  # except 1d tensors
+    MOSTLY_IQ3_ML        = 45  # except 1d tensors
+    MOSTLY_IQ3_XXXL      = 46  # except 1d tensors
+    MOSTLY_IQ3_UXL       = 47  # except 1d tensors
+    MOSTLY_IQ4_XSR       = 48  # except 1d tensors
+    MOSTLY_IQ4_MR        = 49  # except 1d tensors
+    MOSTLY_CQS           = 99  # except 1d tensors
 
     GUESSED              = 1024  # not specified in the model file
 

include/llama.h

@@ -180,12 +180,13 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_Q2_K_L        = 40, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_IQ1_XS        = 41, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_IQ1_XL        = 42, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_IQ4_XSR       = 43, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_IQ3_XXL       = 44, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_XL       = 45, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_IQ3_ML        = 46, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_IQ3_XXXL      = 47, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_IQ3_UXL       = 48, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_XXL       = 43, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q3_K_XL       = 44, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_ML        = 45, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_XXXL      = 46, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ3_UXL       = 47, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ4_XSR       = 48, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_IQ4_MR        = 49, // except 1d tensors
         LLAMA_FTYPE_CQS                  = 99, // except 1d tensors
 
         LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file