Prefetch the backreference hashtable bucket.

Place the prefetch before the last distance checks, to give the prefetch enough time to work. PiperOrigin-RevId: 609192361
google · Apr 12, 2024 · 8461cd9 · 8461cd9
1 parent a76d96e
commit 8461cd9
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 5 deletions.
diff --git a/c/common/platform.h b/c/common/platform.h
@@ -519,6 +519,21 @@ BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) {
 #if BROTLI_ENABLE_DUMP
   BROTLI_UNUSED(&BrotliDump);
 #endif
+
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC)  /* _mm_prefetch() is not defined outside of x86/x64 */
+#  include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#  define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#  define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#  define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#  define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#elif defined(__aarch64__)
+#  define PREFETCH_L1(ptr)  do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0)
+#  define PREFETCH_L2(ptr)  do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0)
+#else
+#  define PREFETCH_L1(ptr) do { (void)(ptr); } while (0)  /* disabled */
+#  define PREFETCH_L2(ptr) do { (void)(ptr); } while (0)  /* disabled */
+#endif
 }
 
 #endif  /* BROTLI_COMMON_PLATFORM_H_ */
diff --git a/c/enc/hash_longest_match64_inc.h b/c/enc/hash_longest_match64_inc.h
@@ -166,6 +166,11 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
   uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
   const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
   BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask)
+  /* Precalculate the hash key and prefetch the bucket. */
+  const size_t key = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
+  uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
+  PREFETCH_L1(bucket);
+  if (self->block_bits_ > 5) PREFETCH_L1(bucket + 16);
   /* Don't accept a short copy from far away. */
   score_t min_score = out->score;
   score_t best_score = out->score;
@@ -217,8 +222,6 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
     best_len = 3;
   }
   {
-    const size_t key = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
-    uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
     const size_t down =
         (num[key] > self->block_size_) ?
         (num[key] - self->block_size_) : 0u;

diff --git a/c/enc/hash_longest_match_inc.h b/c/enc/hash_longest_match_inc.h
@@ -165,6 +165,12 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
   uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
   const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
   BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask)
+  /* Precalculate the hash key and prefetch the bucket. */
+  const uint32_t key =
+      FN(HashBytes)(&data[cur_ix_masked], self->hash_shift_);
+  uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
+  PREFETCH_L1(bucket);
+  if (self->block_bits_ > 5) PREFETCH_L1(bucket + 16);
   /* Don't accept a short copy from far away. */
   score_t min_score = out->score;
   score_t best_score = out->score;
@@ -216,9 +222,6 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
     best_len = 3;
   }
   {
-    const uint32_t key =
-        FN(HashBytes)(&data[cur_ix_masked], self->hash_shift_);
-    uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
     const size_t down =
         (num[key] > self->block_size_) ? (num[key] - self->block_size_) : 0u;
     for (i = num[key]; i > down;) {