POC: Even lower rmse 4-bit Q4_0 quantization

Basically, we use two Q4_0 quantizations, each having 16 weights,
to a quantize a set of 32 weights. We get two separate scaling
factors, which we store as fp16, ending up using the exact same
5 bits per weight as the current Q4_0.

We end up witn an rmse of ~0.00159, so basically the same as
the improved Q4_1. But this should run faster than `Q4_1`
(unless fp16 -> fp32 conversion is somehow very slow).

examples/quantize-stats/quantize-stats.cpp

@@ -307,16 +307,18 @@ int main(int argc, char ** argv) {
 
     // loop throught quantization types
     //for (int i = 0; i < GGML_TYPE_COUNT; i++) {
-    for (int i = 1; i < 2; i++) {
+    for (int i = 0; i < 1; i++) {
         if (!params.include_types.empty() && std::find(params.include_types.begin(), params.include_types.end(), i) == params.include_types.end()) {
             continue;
         }
         quantize_fns_t qfns = ggml_internal_get_quantize_fn(i);
         if (i < 2 && checkNewQuantization) {
             //qfns.quantize_row_q = i == 0 ? kQuantizeQ4_0 : kQuantizeQ4_1;
             //qfns.quantize_row_q = i == 0 ? kQuantizeQ4_0 : kQuantizeQ5_1;
-            qfns.quantize_row_q = i == 0 ? kQuantizeQ4_0 : kQuantizeQ5_1_Fast;
-            if (i == 1) qfns.dequantize_row_q = kDequantizeQ5_1;
+            //qfns.quantize_row_q = i == 0 ? kQuantizeQ4_0 : kQuantizeQ5_1_Fast;
+            //if (i == 1) qfns.dequantize_row_q = kDequantizeQ5_1;
+            qfns.quantize_row_q = i == 0 ? kQuantizeQ4_0K : kQuantizeQ5_1;
+            qfns.dequantize_row_q = i == 0 ? kDequantizeQ4_0K : kDequantizeQ5_1;
         }
         if (qfns.quantize_row_q && qfns.dequantize_row_q) {
             if (params.verbose) {

ggml_extra.cpp

@@ -369,6 +369,15 @@ void kQuantizeQ4(const float* X, void* buffer, int k, int type) {
             std::memcpy(q, &result.second, sizeof(result.second)); q += sizeof(result.second);
             std::memcpy(q, &result.first,  sizeof(result.first));  q += sizeof(result.first);
             for (int k=0; k<QK/2; ++k) q[k] = L[2*k] | (L[2*k+1] << 4);
+        } else if (type == 4) {
+            auto scale1 = quanizeRmseK7(QK/2, X, L);
+            auto scale2 = quanizeRmseK7(QK/2, X+QK/2, L+QK/2);
+            //printf("scale1 = %g, scale2 = %g\n",scale1,scale2);
+            auto scale1fp16 = ggml_fp32_to_fp16(scale1);
+            auto scale2fp16 = ggml_fp32_to_fp16(scale2);
+            std::memcpy(q, &scale1fp16, sizeof(scale1fp16)); q += sizeof(scale1fp16);
+            std::memcpy(q, &scale2fp16, sizeof(scale2fp16)); q += sizeof(scale2fp16);
+            for (int k=0; k<QK/2; ++k) q[k] = (L[2*k] + 8) | ((L[2*k+1] + 8) << 4);
         } else {
             auto result = type == 2 ? kQuantize1(QK, X, L, tmpX, work, 15) : kQuantize1Fast(QK, X, L, 31);
             auto afp16 = ggml_fp32_to_fp16(result.first);
@@ -390,7 +399,7 @@ void kQuantizeQ4(const float* X, void* buffer, int k, int type) {
         }
     };
 
-    auto bucketSize = type == 0 ? kBucketSize0 : kBucketSize1;
+    auto bucketSize = type == 0 || type == 4 ? kBucketSize0 : kBucketSize1;
     auto y = (char*)buffer;
     int nchunk = (k + kChunkSize-1)/kChunkSize;
     if (nchunk < 2) {
@@ -517,4 +526,33 @@ void kDequantizeQ5_1(const void* x, float* y, int k) {
     }
 }
 
+void kQuantizeQ4_0K(const float* x, void* buffer, int k) {
+    kQuantizeQ4(x, buffer, k, 4);
+}
+
+void kDequantizeQ4_0K(const void* x, float* y, int k) {
+    assert(k % QK == 0);
+    int n = k / QK;
+    auto data = (const uint8_t*)x;
+    for (int i=0; i<n; ++i) {
+        ggml_fp16_t afp16, bfp16;
+        std::memcpy(&afp16, data, sizeof(afp16)); data += sizeof(afp16);
+        std::memcpy(&bfp16, data, sizeof(bfp16)); data += sizeof(bfp16);
+        auto a = ggml_fp16_to_fp32(afp16);
+        auto b = ggml_fp16_to_fp32(bfp16);
+        for (int k=0; k<8; ++k) {
+            int8_t l1 = data[k] & 15, l2 = data[k] >> 4;
+            l1 -= 8; l2 -= 8;
+            *y++ = a*l1; *y++ = a*l2;
+        }
+        data += 8;
+        for (int k=0; k<8; ++k) {
+            int8_t l1 = data[k] & 15, l2 = data[k] >> 4;
+            l1 -= 8; l2 -= 8;
+            *y++ = b*l1; *y++ = b*l2;
+        }
+        data += 8;
+    }
+}
+
 }

ggml_extra.h

@@ -28,6 +28,9 @@ void kQuantizeQ5_1_Fast(const float* GGML_RESTRICT x, void* GGML_RESTRICT y, int
 size_t kQuantizeQ5_1H_Fast(const float* GGML_RESTRICT x, void* GGML_RESTRICT y, int k, int64_t* hist);
 void kDequantizeQ5_1(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);
 
+void kQuantizeQ4_0K(const float* GGML_RESTRICT x, void* GGML_RESTRICT y, int k);
+void kDequantizeQ4_0K(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);
+
 #ifdef  __cplusplus
 }
 #endif