From 79e14129e1c5aa31d4f5455f36156598f9f0efe6 Mon Sep 17 00:00:00 2001
From: Sergey Pershukov <perserk@gmail.com>
Date: Thu, 30 Mar 2023 09:43:27 +0500
Subject: [PATCH 1/3] ggml : add AVX quantize_row_q4_0()
---
ggml.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/ggml.c b/ggml.c
index 02675ee67072d..d9ae20140f14c 100644
--- a/ggml.c
+++ b/ggml.c
@@ -461,6 +461,22 @@ static inline __m128i packNibbles( __m256i bytes )
__m128i r1 = _mm256_extracti128_si256( bytes, 1 );
return _mm_packus_epi16( r0, r1 );
}
+#elif __AVX__
+static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )
+{
+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
+ const __m128i lowByte = _mm_set1_epi16( 0xFF );
+ __m128i high = _mm_andnot_si128( lowByte, bytes1 );
+ __m128i low = _mm_and_si128( lowByte, bytes1 );
+ high = _mm_srli_epi16( high, 4 );
+ bytes1 = _mm_or_si128( low, high );
+ high = _mm_andnot_si128( lowByte, bytes2 );
+ low = _mm_and_si128( lowByte, bytes2 );
+ high = _mm_srli_epi16( high, 4 );
+ bytes2 = _mm_or_si128( low, high );
+
+ return _mm_packus_epi16( bytes1, bytes2);
+}
#endif
// method 5
@@ -660,6 +676,80 @@ static void quantize_row_q4_0(const float * restrict x, void * restrict vy, int
__m128i res = packNibbles( i0 );
_mm_storeu_si128( ( __m128i* )y[i].qs, res );
}
+#elif defined(__AVX__)
+ for (int i = 0; i < nb; i++) {
+ // Load elements into 4 AVX vectors
+ __m256 v0 = _mm256_loadu_ps( x );
+ __m256 v1 = _mm256_loadu_ps( x + 8 );
+ __m256 v2 = _mm256_loadu_ps( x + 16 );
+ __m256 v3 = _mm256_loadu_ps( x + 24 );
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 signBit = _mm256_set1_ps( -0.0f );
+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );
+
+ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );
+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );
+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );
+ const float maxScalar = _mm_cvtss_f32( max4 );
+
+ // Quantize these floats
+ const float d = maxScalar / 7.0f;
+ y[i].d = d;
+ const float id = ( maxScalar != 0.0f ) ? 7.0f / maxScalar : 0.0f;
+ const __m256 mul = _mm256_set1_ps( id );
+
+ // Apply the multiplier
+ v0 = _mm256_mul_ps( v0, mul );
+ v1 = _mm256_mul_ps( v1, mul );
+ v2 = _mm256_mul_ps( v2, mul );
+ v3 = _mm256_mul_ps( v3, mul );
+
+ // Round to nearest integer
+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );
+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );
+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );
+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );
+
+ // Convert floats to integers
+ __m256i i0 = _mm256_cvtps_epi32( v0 );
+ __m256i i1 = _mm256_cvtps_epi32( v1 );
+ __m256i i2 = _mm256_cvtps_epi32( v2 );
+ __m256i i3 = _mm256_cvtps_epi32( v3 );
+
+ // Since we don't have in AVX some necessary functions,
+ // we split the registers in half and call AVX2 analogs from SSE
+ __m128i ni0 = _mm256_castsi256_si128( i0 );
+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);
+ __m128i ni2 = _mm256_castsi256_si128( i1 );
+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);
+ __m128i ni4 = _mm256_castsi256_si128( i2 );
+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);
+ __m128i ni6 = _mm256_castsi256_si128( i3 );
+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);
+
+ // Convert int32 to int16
+ ni0 = _mm_packs_epi32( ni0, ni1 );
+ ni2 = _mm_packs_epi32( ni2, ni3 );
+ ni4 = _mm_packs_epi32( ni4, ni5 );
+ ni6 = _mm_packs_epi32( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = _mm_packs_epi16( ni0, ni2 );
+ ni4 = _mm_packs_epi16( ni4, ni6 );
+
+ // Apply offset to translate the range from [ -7 .. +7 ] into [ +1 .. +15 ]
+ const __m128i off = _mm_set1_epi8( 8);
+ ni0 = _mm_add_epi8( ni0, off );
+ ni4 = _mm_add_epi8( ni4, off );
+
+ // Compress the vector into 4 bit/value, and store
+ __m128i res = packNibbles( ni0, ni4 );
+ _mm_storeu_si128( ( __m128i* )y[i].qs, res );
+ }
#elif defined(__wasm_simd128__)
for (int i = 0; i < nb; i++) {
float amax = 0.0f; // absolute max
From 93a316928450a5dcd3c4d91d1f012b6f67c4dc59 Mon Sep 17 00:00:00 2001
From: Sergey Pershukov <perserk@gmail.com>
Date: Thu, 30 Mar 2023 09:50:40 +0500
Subject: [PATCH 2/3] ggml : add AVX ggml_vec_dot_q4_0()
---
ggml.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 63 insertions(+)
diff --git a/ggml.c b/ggml.c
index d9ae20140f14c..7f1c4409d487a 100644
--- a/ggml.c
+++ b/ggml.c
@@ -462,6 +462,23 @@ static inline __m128i packNibbles( __m256i bytes )
return _mm_packus_epi16( r0, r1 );
}
#elif __AVX__
+static inline __m128i bytesFromNibbles( const uint8_t* rsi )
+{
+ // Load 8 bytes from memory
+ __m128i tmp = _mm_loadu_si64( ( const __m128i* )rsi );
+
+ // Expand bytes into uint16_t values
+ __m128i bytes = _mm_cvtepu8_epi16( tmp );
+
+ // Unpack values into individual bytes
+ const __m128i lowMask = _mm_set1_epi8( 0xF );
+ __m128i high = _mm_andnot_si128( lowMask, bytes );
+ __m128i low = _mm_and_si128( lowMask, bytes );
+ high = _mm_slli_epi16( high, 4 );
+ bytes = _mm_or_si128( low, high );
+ return bytes;
+}
+
static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )
{
// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
@@ -1983,6 +2000,52 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest
res = _mm_add_ps( res, _mm_movehl_ps( res, res ) );
res = _mm_add_ss( res, _mm_movehdup_ps( res ) );
+ sumf = _mm_cvtss_f32( res );
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (int i = 0; i < nb; ++i) {
+ // Compute combined scale for the block
+ const __m256 d = _mm256_mul_ps( _mm256_broadcast_ss( &x[i].d ), _mm256_broadcast_ss( &y[i].d ) );
+
+ __m128i i32[2];
+ for (int j = 0; j < 2; ++j) {
+ // Load 8 bytes, and unpack 4 bit fields into bytes, making 16 bytes
+ __m128i bx = bytesFromNibbles( x[i].qs + 8*j );
+ __m128i by = bytesFromNibbles( y[i].qs + 8*j );
+
+ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
+ const __m128i off = _mm_set1_epi8( 8 );
+ bx = _mm_sub_epi8( bx, off );
+ by = _mm_sub_epi8( by, off );
+
+ // Sign-extend first 8 signed bytes into int16_t
+ __m128i x16 = _mm_cvtepi8_epi16( bx );
+ __m128i y16 = _mm_cvtepi8_epi16( by );
+ // Compute products of int16_t integers, add pairwise
+ i32[j] = _mm_madd_epi16( x16, y16 );
+
+ // Sign-extend last 8 signed bytes into int16_t vectors
+ x16 = _mm_cvtepi8_epi16( _mm_srli_si128( bx, 8 ) );
+ y16 = _mm_cvtepi8_epi16( _mm_srli_si128( by, 8 ) );
+ // Accumulate products of int16_t integers
+ i32[j] = _mm_add_epi32( i32[j], _mm_madd_epi16( x16, y16 ) );
+ }
+
+ // Convert int32_t to float
+ __m256 p = _mm256_cvtepi32_ps( _mm256_set_m128i( i32[0], i32[1] ));
+ // Apply the scale, and accumulate
+ acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);
+ }
+
+ // Return horizontal sum of the acc vector
+ __m128 res = _mm256_extractf128_ps( acc, 1 );
+ res = _mm_add_ps( res, _mm256_castps256_ps128( acc ) );
+ res = _mm_add_ps( res, _mm_movehl_ps( res, res ) );
+ res = _mm_add_ss( res, _mm_movehdup_ps( res ) );
+
sumf = _mm_cvtss_f32( res );
#elif defined(__wasm_simd128__)
// wasm simd
From 80dad7923e5b52d0ee06c7baf2dc6febc07ca433 Mon Sep 17 00:00:00 2001
From: Sergey Pershukov <perserk@gmail.com>
Date: Thu, 30 Mar 2023 21:44:34 +0500
Subject: [PATCH 3/3] ggml : refactor AVX part of ggml_vec_dot_q4_0()
https://github.com/ggerganov/llama.cpp/pull/617#issuecomment-1489985645
---
ggml.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/ggml.c b/ggml.c
index 7f1c4409d487a..05710559a41ab 100644
--- a/ggml.c
+++ b/ggml.c
@@ -2021,17 +2021,17 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest
bx = _mm_sub_epi8( bx, off );
by = _mm_sub_epi8( by, off );
- // Sign-extend first 8 signed bytes into int16_t
- __m128i x16 = _mm_cvtepi8_epi16( bx );
- __m128i y16 = _mm_cvtepi8_epi16( by );
- // Compute products of int16_t integers, add pairwise
- i32[j] = _mm_madd_epi16( x16, y16 );
-
- // Sign-extend last 8 signed bytes into int16_t vectors
- x16 = _mm_cvtepi8_epi16( _mm_srli_si128( bx, 8 ) );
- y16 = _mm_cvtepi8_epi16( _mm_srli_si128( by, 8 ) );
- // Accumulate products of int16_t integers
- i32[j] = _mm_add_epi32( i32[j], _mm_madd_epi16( x16, y16 ) );
+ // Get absolute values of x vectors
+ const __m128i ax = _mm_sign_epi8(bx, bx);
+
+ // Sign the values of the y vectors
+ const __m128i sy = _mm_sign_epi8(by, bx);
+
+ // Perform multiplication and create 16-bit values
+ const __m128i dot = _mm_maddubs_epi16(ax, sy);
+
+ const __m128i ones = _mm_set1_epi16(1);
+ i32[j] = _mm_madd_epi16(ones, dot);
}
// Convert int32_t to float