From 42e7161376f29e985977656f89485a14725212ad Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 00:29:30 +0000 Subject: [PATCH] Optimize decompress_bmi2 for offset 3 using SIMD This commit optimizes the decompression of `offset == 3` matches in `decompress_bmi2` (x86-64). Previously, this case fell back to a slow byte-by-byte scalar copy loop. The optimization uses `_mm_shuffle_epi8` with precomputed cyclic masks (`OFFSET3_MASKS`) to construct 16-byte vectors containing the repeating 3-byte pattern from the first 16 bytes loaded from `src` (safely masking out garbage). The copy loop is unrolled to process 48 bytes (LCM of 3 and 16) per iteration. Performance Impact: - `Decompress offset3` throughput improved by ~540% (~1.44 GiB/s -> ~9.22 GiB/s). - `Decompress offset3 small` throughput improved by ~1.4%. - `Decompress offset30` throughput improved by ~4%. - `Decompress offset31` throughput improved by ~9%. - `Decompress offset32` throughput improved by ~1.8%. This aligns the performance of offset 3 with other small offsets that are already optimized. Co-authored-by: 404Setup <153366651+404Setup@users.noreply.github.com> --- .jules/bolt.md | 4 +++ src/decompress/x86.rs | 64 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index 78e0029..576d86b 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -27,3 +27,7 @@ ## 2026-06-04 - [Vector Precomputation vs Alignr Chain] **Learning:** For overlapping patterns where offset is a multiple of 8 (e.g., offset 24), breaking the `alignr` dependency chain by precomputing all vectors in the cycle (LCM of offset and vector size) allowed for effective loop unrolling. This yielded a 32% throughput improvement (7.7 GiB/s -> 10.2 GiB/s) by increasing ILP compared to the serial dependency of iterative `alignr`. **Action:** When optimizing decompression loops for specific offsets, determine if the pattern cycle is short enough to precompute fully. If so, prefer storing precomputed vectors in an unrolled loop over calculating the next vector from the previous one. + +## 2026-06-04 - [Offset 3 Optimization] +**Learning:** Decompressing `offset == 3` using a byte-by-byte scalar loop is extremely slow (~1.44 GiB/s). By using precomputed shuffle masks (3 vectors for the 48-byte cycle, LCM(3, 16)) and loading 16 bytes from `src` (safely masking out garbage), we can process 48 bytes per iteration using SIMD. This yielded a ~540% throughput improvement (~9.2 GiB/s). +**Action:** For small offsets (like 3) that don't fit power-of-2 optimizations, use `pshufb` with precomputed cyclic masks to construct the pattern vectors. diff --git a/src/decompress/x86.rs b/src/decompress/x86.rs index c61abbb..cff2d49 100644 --- a/src/decompress/x86.rs +++ b/src/decompress/x86.rs @@ -49,6 +49,12 @@ static OFFSET12_MASKS: [u8; 48] = [ 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, ]; +// LCM(3, 16) = 48. 3 vectors. +static OFFSET3_MASKS: [u8; 48] = [ + 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, + 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, +]; + // LCM(10, 16) = 80. 5 vectors. static OFFSET10_MASKS: [u8; 80] = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, @@ -931,8 +937,64 @@ pub unsafe fn decompress_bmi2( *out_next.add(i) = (pattern >> ((i & 7) * 8)) as u8; i += 1; } + } else if offset == 3 { + let dest_ptr = out_next; + let src_ptr = src; + let mut copied = 0; + + if length >= 16 { + let v_src = _mm_loadu_si128(src_ptr as *const __m128i); + let masks_ptr = + OFFSET3_MASKS.as_ptr() as *const __m128i; + let v_base = + _mm_shuffle_epi8(v_src, _mm_loadu_si128(masks_ptr)); + + while copied + 48 <= length { + _mm_storeu_si128( + dest_ptr.add(copied) as *mut __m128i, + v_base, + ); + _mm_storeu_si128( + dest_ptr.add(copied + 16) as *mut __m128i, + _mm_shuffle_epi8( + v_src, + _mm_loadu_si128(masks_ptr.add(1)), + ), + ); + _mm_storeu_si128( + dest_ptr.add(copied + 32) as *mut __m128i, + _mm_shuffle_epi8( + v_src, + _mm_loadu_si128(masks_ptr.add(2)), + ), + ); + copied += 48; + } + + while copied + 16 <= length { + let idx = (copied % 48) / 16; + let v = if idx == 0 { + v_base + } else { + _mm_shuffle_epi8( + v_src, + _mm_loadu_si128(masks_ptr.add(idx)), + ) + }; + _mm_storeu_si128( + dest_ptr.add(copied) as *mut __m128i, + v, + ); + copied += 16; + } + } + + while copied < length { + *dest_ptr.add(copied) = *src_ptr.add(copied); + copied += 1; + } } else { - // Simple loop for offsets 3, 5, 6, 7 + // Simple loop for offsets 5, 6, 7 let mut copied = 0; while copied < length { *out_next.add(copied) = *src.add(copied);