From ca01960e6a30e3d6a4cfeb1755d6e027b31f6b07 Mon Sep 17 00:00:00 2001 From: pl752 Date: Sun, 19 Apr 2026 00:09:03 -0700 Subject: [PATCH 01/11] ggml-cpu: Optimized x86 and generic cpu q1_0 dot (follow up) (#21636) --- ggml/src/ggml-cpu/arch-fallback.h | 1 - ggml/src/ggml-cpu/arch/arm/quants.c | 30 +----- ggml/src/ggml-cpu/arch/x86/quants.c | 158 ++++++++++++++++++++++++++++ ggml/src/ggml-cpu/quants.c | 26 +++-- 4 files changed, 179 insertions(+), 36 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index c589a213e9d4..595ded09f03f 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -83,7 +83,6 @@ #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index 64d811fafe71..fe6213329708 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -151,8 +151,6 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const block_q1_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; - float sumf = 0.0f; - #if defined(__ARM_NEON) float32x4_t sumv = vdupq_n_f32(0.0f); @@ -212,31 +210,13 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } } - sumf = vaddvq_f32(sumv); + *s = vaddvq_f32(sumv); #else - // Scalar fallback - for (int i = 0; i < nb; i++) { - const float d0 = GGML_FP16_TO_FP32(x[i].d); - - // Process 4 Q8_0 blocks - for (int k = 0; k < 4; k++) { - const float d1 = GGML_FP16_TO_FP32(y[i*4 + k].d); - - int sumi = 0; - for (int j = 0; j < QK8_0; j++) { - const int bit_index = k * QK8_0 + j; - const int byte_index = bit_index / 8; - const int bit_offset = bit_index % 8; - - const int xi = ((x[i].qs[byte_index] >> bit_offset) & 1) ? 1 : -1; - sumi += xi * y[i*4 + k].qs[j]; - } - sumf += d0 * d1 * sumi; - } - } + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); #endif - - *s = sumf; } diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 74d699f633d3..0a3e071e57c9 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -274,6 +274,18 @@ static inline __m256 quad_mx_delta_float(const uint8_t x0, const float y0, const } #endif #elif defined(__SSSE3__) +static inline __m128i bytes_from_bits_16(const uint8_t * x) { + uint16_t x16; + memcpy(&x16, x, sizeof(uint16_t)); + + const __m128i shuf_mask = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000); + __m128i bytes = _mm_shuffle_epi8(_mm_set1_epi16((short) x16), shuf_mask); + const __m128i bit_mask = _mm_set_epi64x(0x7fbfdfeff7fbfdfe, 0x7fbfdfeff7fbfdfe); + bytes = _mm_or_si128(bytes, bit_mask); + + return _mm_cmpeq_epi8(bytes, _mm_set1_epi64x(-1)); +} + // horizontally add 4x4 floats static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) { __m128 res_0 =_mm_hadd_ps(a, b); @@ -540,6 +552,152 @@ static inline __m128i get_scale_shuffle(int i) { } #endif +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__AVX2__) + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i byte_shuf = _mm256_setr_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3); + const __m256i bit_masks = _mm256_setr_epi8( + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128, + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128); + const __m256i zero = _mm256_setzero_si256(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const uint32_t * GGML_RESTRICT qs32 = (const uint32_t *) x[ib].qs; + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + + __m256 acc_block; + { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[0].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[0]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), _mm256_cvtepi32_ps(s32)); + } + for (int K = 1; K < 4; ++K) { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[K].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[K]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[K].d)), _mm256_cvtepi32_ps(s32), acc_block); + } + acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); + } + + *s = hsum_float_8(acc); +#elif defined(__AVX__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + __m256 acc_block; + { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[0]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), q); + } + for(int K = 1; K < 4; ++K) { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[(K) * 4]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_add_ps(acc_block, _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(K)].d)), q)); + } +#undef Q1_AVX_BLOCK + + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_set1_ps(d0), acc_block)); + } + + *s = hsum_float_8(acc); +#elif defined(__SSSE3__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m128 acc_0 = _mm_setzero_ps(); + __m128 acc_1 = _mm_setzero_ps(); + __m128 acc_2 = _mm_setzero_ps(); + __m128 acc_3 = _mm_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const __m128 d0 = _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d)); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + +#define Q1_SSSE3_BLOCK(QS_OFF, Y_IDX, ACC) \ + { \ + const __m128i bit_mask_0 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 0]); \ + const __m128i bit_mask_1 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 2]); \ + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[0]); \ + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[16]); \ + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); \ + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); \ + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); \ + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); \ + const __m128i sum_0 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_0), ones_16); \ + const __m128i sum_1 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_1), ones_16); \ + const __m128 q = _mm_cvtepi32_ps(_mm_add_epi32(sum_0, sum_1)); \ + (ACC) = _mm_add_ps((ACC), _mm_mul_ps(_mm_mul_ps(d0, _mm_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(Y_IDX)].d))), q)); \ + } + Q1_SSSE3_BLOCK(0, 0, acc_0) + Q1_SSSE3_BLOCK(4, 1, acc_1) + Q1_SSSE3_BLOCK(8, 2, acc_2) + Q1_SSSE3_BLOCK(12, 3, acc_3) +#undef Q1_SSSE3_BLOCK + } + + *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index f66127c22909..e5f9a4083f9c 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -137,22 +137,28 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c float sumf = 0.0; for (int i = 0; i < nb; i++) { - const float d0 = GGML_FP16_TO_FP32(x[i].d); + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); float sumi = 0.0f; for (int k = 0; k < 4; k++) { - const float d1 = GGML_FP16_TO_FP32(y[i*4 + k].d); - + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); int sumi_block = 0; - for (int j = 0; j < QK8_0; j++) { - const int bit_index = k * QK8_0 + j; - const int byte_index = bit_index / 8; - const int bit_offset = bit_index % 8; - - const int xi = ((x[i].qs[byte_index] >> bit_offset) & 1) ? 1 : -1; - sumi_block += xi * y[i*4 + k].qs[j]; + const uint8_t * GGML_RESTRICT bits = &x[i].qs[k * 4]; + const int8_t * GGML_RESTRICT qy = yb->qs; + + for (int b = 0; b < 4; ++b, qy += 8) { + const unsigned mask = bits[b]; + sumi_block += ((mask & 0x01) ? qy[0] : -qy[0]) + + ((mask & 0x02) ? qy[1] : -qy[1]) + + ((mask & 0x04) ? qy[2] : -qy[2]) + + ((mask & 0x08) ? qy[3] : -qy[3]) + + ((mask & 0x10) ? qy[4] : -qy[4]) + + ((mask & 0x20) ? qy[5] : -qy[5]) + + ((mask & 0x40) ? qy[6] : -qy[6]) + + ((mask & 0x80) ? qy[7] : -qy[7]); } sumi += d1 * sumi_block; From 7e2a21808a10a7de793bdcd60e39c627a25b016f Mon Sep 17 00:00:00 2001 From: Pasha Khosravi Date: Sun, 19 Apr 2026 00:09:11 -0700 Subject: [PATCH 02/11] Add release-prism workflow --- .github/workflows/release-prism.yml | 742 ++++++++++++++++++++++++++++ 1 file changed, 742 insertions(+) create mode 100644 .github/workflows/release-prism.yml diff --git a/.github/workflows/release-prism.yml b/.github/workflows/release-prism.yml new file mode 100644 index 000000000000..cc9fb008f7cd --- /dev/null +++ b/.github/workflows/release-prism.yml @@ -0,0 +1,742 @@ +name: Release (Prism) + +on: + workflow_dispatch: + inputs: + create_release: + description: 'Create new release' + required: true + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON" + +jobs: + macOS-cpu: + strategy: + matrix: + include: + - build: 'arm64' + arch: 'arm64' + os: macos-14 + defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON" + - build: 'arm64-kleidiai' + arch: 'arm64' + os: macos-14 + defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_CPU_KLEIDIAI=ON" + - build: 'x64' + arch: 'x64' + os: macos-15-intel + defines: "-DGGML_METAL=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" + + runs-on: ${{ matrix.os }} + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: macOS-latest-${{ matrix.arch }} + evict-old-files: 1d + + - name: Build + run: | + sysctl -a + cmake -B build \ + ${{ matrix.defines }} \ + -DCMAKE_INSTALL_RPATH='@loader_path' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz + name: llama-bin-macos-${{ matrix.build }}.tar.gz + + ubuntu-cpu: + strategy: + matrix: + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-24.04-arm + + runs-on: ${{ matrix.os }} + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: ubuntu-cpu-${{ matrix.build }} + evict-old-files: 1d + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install build-essential libssl-dev + + - name: Toolchain workaround (GCC 14) + if: ${{ contains(matrix.os, 'ubuntu-24.04') }} + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: Build + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz + + linux-cuda: + runs-on: ubuntu-22.04 + + strategy: + matrix: + include: + - cuda: '12.4' + cuda_pkg: '12-4' + - cuda: '12.8' + cuda_pkg: '12-8' + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: ubuntu-22-cmake-cuda-${{ matrix.cuda }} + evict-old-files: 1d + + - name: Install CUDA toolkit + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get update + sudo apt-get -y install cuda-toolkit-${{ matrix.cuda_pkg }} + echo "/usr/local/cuda-${{ matrix.cuda }}/bin" >> $GITHUB_PATH + echo "CUDA_PATH=/usr/local/cuda-${{ matrix.cuda }}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/local/cuda-${{ matrix.cuda }}/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Build + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) 2>&1 | grep -v "^nvcc warning" + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-linux-cuda-${{ matrix.cuda }}-x64.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-linux-cuda-${{ matrix.cuda }}-x64.tar.gz + name: llama-bin-linux-cuda-${{ matrix.cuda }}-x64.tar.gz + + windows-cpu: + runs-on: windows-2025 + + strategy: + matrix: + include: + - arch: 'x64' + - arch: 'arm64' + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: windows-latest-cpu-${{ matrix.arch }} + variant: ccache + evict-old-files: 1d + + - name: Install Ninja + run: choco install ninja + + - name: Build + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }} + cmake -S . -B build -G "Ninja Multi-Config" ^ + -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_NATIVE=OFF ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^ + -DGGML_OPENMP=ON ^ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release + + - name: Pack artifacts + run: | + Copy-Item "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC\14.44.35112\debug_nonredist\${{ matrix.arch }}\Microsoft.VC143.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\ + 7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-cpu-${{ matrix.arch }}.zip + name: llama-bin-win-cpu-${{ matrix.arch }}.zip + + windows-cuda: + runs-on: windows-2022 + + strategy: + matrix: + cuda: ['12.4'] + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: Install ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: windows-cuda-${{ matrix.cuda }} + variant: ccache + evict-old-files: 1d + + - name: Install Cuda Toolkit + uses: ./.github/actions/windows-setup-cuda + with: + cuda_version: ${{ matrix.cuda }} + + - name: Install Ninja + run: choco install ninja + + - name: Build + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + cmake -S . -B build -G "Ninja Multi-Config" ^ + -DGGML_NATIVE=OFF ^ + -DGGML_CUDA=ON ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DCMAKE_CUDA_FLAGS="-diag-suppress=221" ^ + ${{ env.CMAKE_ARGS }} + set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 + cmake --build build --config Release -j %NINJA_JOBS% + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-win-cuda-${{ matrix.cuda }}-x64.zip + name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + + - name: Copy and pack Cuda runtime + run: | + echo "Cuda install location: ${{ env.CUDA_PATH }}" + $dst='.\build\bin\cudart\' + robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + robocopy "${{env.CUDA_PATH}}\bin\x64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\* + + - name: Upload Cuda runtime + uses: actions/upload-artifact@v6 + with: + path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + + ubuntu-vulkan: + strategy: + matrix: + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-24.04-arm + + runs-on: ${{ matrix.os }} + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: ubuntu-vulkan-${{ matrix.build }} + evict-old-files: 1d + + - name: Dependencies + run: | + if [[ "${{ matrix.os }}" =~ "ubuntu-22.04" ]]; then + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - + sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + sudo apt-get update -y + sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev + else + sudo apt-get update -y + sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev libssl-dev ninja-build + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + fi + + - name: Build + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_VULKAN=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz + + windows-vulkan: + runs-on: windows-2025 + + env: + VULKAN_VERSION: 1.4.313.2 + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: windows-latest-vulkan-x64 + variant: ccache + evict-old-files: 1d + + - name: Install Vulkan SDK + run: | + curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe" + & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install + Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" + Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" + + - name: Install Ninja + run: choco install ninja + + - name: Build + run: | + cmake -S . -B build -DGGML_VULKAN=ON -DGGML_NATIVE=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --config Release --target ggml-vulkan + + - name: Pack artifacts + run: | + 7z a -snl llama-bin-win-vulkan-x64.zip .\build\bin\Release\ggml-vulkan.dll + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-vulkan-x64.zip + name: llama-bin-win-vulkan-x64.zip + + ubuntu-22-rocm: + runs-on: ubuntu-22.04 + + strategy: + matrix: + include: + - ROCM_VERSION: "7.2.1" + gpu_targets: "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201" + build: 'x64' + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }} + evict-old-files: 1d + + - name: Dependencies + run: | + sudo apt install -y build-essential git cmake wget + + - name: Setup Legacy ROCm + if: matrix.ROCM_VERSION == '7.2.1' + run: | + sudo mkdir --parents --mode=0755 /etc/apt/keyrings + wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ + gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null + + sudo tee /etc/apt/sources.list.d/rocm.list << EOF + deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ matrix.ROCM_VERSION }} jammy main + EOF + + sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF + Package: * + Pin: release o=repo.radeon.com + Pin-Priority: 600 + EOF + + sudo apt update + sudo apt-get install -y libssl-dev rocm-hip-sdk + + - name: Build with native CMake HIP support + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGPU_TARGETS="${{ matrix.gpu_targets }}" \ + -DGGML_HIP=ON \ + -DHIP_PLATFORM=amd \ + -DGGML_HIP_ROCWMMA_FATTN=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Get ROCm short version + run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV + + - name: Pack artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + + windows-hip: + runs-on: windows-2022 + + env: + HIPSDK_INSTALLER_VERSION: "26.Q1" + + strategy: + matrix: + include: + - name: "radeon" + gpu_targets: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032" + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: Grab rocWMMA package + run: | + curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb" + 7z x rocwmma.deb + 7z x data.tar + + - name: Cache ROCm Installation + id: cache-rocm + uses: actions/cache@v5 + with: + path: C:\Program Files\AMD\ROCm + key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }} + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: windows-latest-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }}-x64 + evict-old-files: 1d + + - name: Install ROCm + if: steps.cache-rocm.outputs.cache-hit != 'true' + run: | + $ErrorActionPreference = "Stop" + write-host "Downloading AMD HIP SDK Installer" + Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe" + write-host "Installing AMD HIP SDK" + $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru + $completed = $proc.WaitForExit(600000) + if (-not $completed) { + Write-Error "ROCm installation timed out after 10 minutes. Killing the process" + $proc.Kill() + exit 1 + } + if ($proc.ExitCode -ne 0) { + Write-Error "ROCm installation failed with exit code $($proc.ExitCode)" + exit 1 + } + write-host "Completed AMD HIP SDK installation" + + - name: Verify ROCm + run: | + $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1 + if (-not $clangPath) { + Write-Error "ROCm installation not found" + exit 1 + } + & $clangPath.FullName --version + + - name: Build + run: | + $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path) + $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}" + cmake -G "Unix Makefiles" -B build -S . ` + -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" ` + -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" ` + -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" ` + -DCMAKE_BUILD_TYPE=Release ` + -DGGML_BACKEND_DL=ON ` + -DGGML_NATIVE=OFF ` + -DGGML_CPU=OFF ` + -DGPU_TARGETS="${{ matrix.gpu_targets }}" ` + -DGGML_HIP_ROCWMMA_FATTN=ON ` + -DGGML_HIP=ON ` + -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --target ggml-hip -j ${env:NUMBER_OF_PROCESSORS} + md "build\bin\rocblas\library\" + md "build\bin\hipblaslt\library" + cp "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\" + cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\" + + - name: Pack artifacts + run: | + 7z a -snl llama-bin-win-hip-${{ matrix.name }}-x64.zip .\build\bin\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-hip-${{ matrix.name }}-x64.zip + name: llama-bin-win-hip-${{ matrix.name }}-x64.zip + + ios-xcode-build: + runs-on: macos-15 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Xcode + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app + + - name: Build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_USE_BF16=ON \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + + - name: xcodebuild for swift package + run: | + ./build-xcframework.sh + + - name: Build Xcode project + run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + run: | + zip -r -y llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-xcframework.zip + name: llama-${{ steps.tag.outputs.name }}-xcframework.zip + + release: + if: ${{ github.event.inputs.create_release == 'true' }} + + permissions: + contents: write + + runs-on: ubuntu-latest + + needs: + - macOS-cpu + - ubuntu-cpu + - ubuntu-vulkan + - linux-cuda + - ubuntu-22-rocm + - windows-cpu + - windows-vulkan + - windows-cuda + - windows-hip + - ios-xcode-build + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Download artifacts + uses: actions/download-artifact@v7 + with: + path: ./artifact + merge-multiple: true + + - name: Move artifacts + run: | + mkdir -p release + mv -v artifact/*.tar.gz release/ 2>/dev/null || true + mv -v artifact/*.zip release/ 2>/dev/null || true + ls -lh release/ + + - name: Create release + id: create_release + uses: ggml-org/action-create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.tag.outputs.name }} + body: | + Pre-built binaries (PrismML fork with Q1_0 1-bit quantization support). + + **macOS/iOS:** + - [macOS Apple Silicon (arm64)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz) + - [macOS Apple Silicon (arm64, KleidiAI enabled)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64-kleidiai.tar.gz) + - [macOS Intel (x64)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz) + - [iOS XCFramework](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip) + + **Linux (CPU):** + - [Ubuntu x64 (CPU)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz) + - [Ubuntu arm64 (CPU)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz) + + **Linux (CUDA):** + - [Linux x64 (CUDA 12.4)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-cuda-12.4-x64.tar.gz) + - [Linux x64 (CUDA 12.8)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-cuda-12.8-x64.tar.gz) + + **Windows (CPU):** + - [Windows x64 (CPU)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-bin-win-cpu-x64.zip) + - [Windows arm64 (CPU)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-bin-win-cpu-arm64.zip) + + **Linux (Vulkan):** + - [Ubuntu x64 (Vulkan)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) + - [Ubuntu arm64 (Vulkan)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) + + **Linux (AMD):** + - [Ubuntu x64 (ROCm 7.2)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.2-x64.tar.gz) + + **Windows (CUDA):** + - [Windows x64 (CUDA 12.4)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) + - [Windows x64 (Vulkan)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-bin-win-vulkan-x64.zip) + - [Windows x64 (HIP/ROCm)](https://github.com/${{ github.repository }}/releases/download/${{ steps.tag.outputs.name }}/llama-bin-win-hip-radeon-x64.zip) + + - name: Upload release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in release/*; do + echo "Uploading $(basename $file)..." + gh release upload ${{ steps.tag.outputs.name }} "$file" --clobber + done From 9f31ffca2e2216408c71db0da6edcb7082135d08 Mon Sep 17 00:00:00 2001 From: Pasha Khosravi Date: Fri, 17 Apr 2026 21:34:00 -0700 Subject: [PATCH 03/11] Add Q2_0 quantization: type definition and CPU backend --- convert_hf_to_gguf.py | 3 ++ ggml/include/ggml.h | 4 +- ggml/src/ggml-common.h | 10 ++++ ggml/src/ggml-cpu/arch/arm/quants.c | 73 +++++++++++++++++++++++++++ ggml/src/ggml-cpu/ggml-cpu.c | 6 +++ ggml/src/ggml-cpu/ops.cpp | 7 +++ ggml/src/ggml-cpu/quants.c | 50 +++++++++++++++++++ ggml/src/ggml-cpu/quants.h | 3 ++ ggml/src/ggml-quants.c | 76 +++++++++++++++++++++++++++++ ggml/src/ggml-quants.h | 3 ++ ggml/src/ggml.c | 10 ++++ gguf-py/gguf/constants.py | 3 ++ gguf-py/gguf/quants.py | 39 +++++++++++++++ include/llama.h | 1 + src/llama-model-loader.cpp | 2 + src/llama-quant.cpp | 4 +- tools/quantize/quantize.cpp | 1 + 17 files changed, 293 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2df5e94fe236..508b1afe3826 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -869,6 +869,8 @@ def prepare_tensors(self): data_qtype = gguf.GGMLQuantizationType.TQ1_0 elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0: data_qtype = gguf.GGMLQuantizationType.TQ2_0 + elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_0: + data_qtype = gguf.GGMLQuantizationType.Q2_0 else: raise ValueError(f"Unknown file type: {self.ftype.name}") @@ -13407,6 +13409,7 @@ def main() -> None: "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0, "tq1_0": gguf.LlamaFileType.MOSTLY_TQ1_0, "tq2_0": gguf.LlamaFileType.MOSTLY_TQ2_0, + "q2_0": gguf.LlamaFileType.MOSTLY_Q2_0, "auto": gguf.LlamaFileType.GUESSED, } diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 703e37831361..2adf0652abe4 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -429,7 +429,8 @@ extern "C" { GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, - GGML_TYPE_COUNT = 42, + GGML_TYPE_Q2_0 = 42, + GGML_TYPE_COUNT = 43, }; // precision @@ -467,6 +468,7 @@ extern "C" { GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors }; // available tensor operations: diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index f05683b44cd9..3bccec15c289 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -96,6 +96,9 @@ typedef sycl::half2 ggml_half2; #define QI1_0 (QK1_0 / 32) #define QR1_0 1 +#define QI2_0 (QK2_0 / 32) +#define QR2_0 1 + #define QI4_0 (QK4_0 / (4 * QR4_0)) #define QR4_0 2 @@ -181,6 +184,13 @@ typedef struct { } block_q1_0; static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding"); +#define QK2_0 128 +typedef struct { + ggml_half d; // delta (scale) + uint8_t qs[QK2_0 / 4]; // 2 bits per element +} block_q2_0; +static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); + #define QK4_0 32 typedef struct { ggml_half d; // delta diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index fe6213329708..786bd819100e 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -219,6 +219,79 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + // Replicate pattern: each byte repeated 4 times + static const uint8_t tbl_idx_lo[16] = {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3}; + static const uint8_t tbl_idx_hi[16] = {4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7}; + // Right-shift amounts: 0,2,4,6 repeated for each group of 4 + static const int8_t shift_vals[16] = {0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6}; + + const uint8x16_t idx_lo = vld1q_u8(tbl_idx_lo); + const uint8x16_t idx_hi = vld1q_u8(tbl_idx_hi); + const int8x16_t shifts = vld1q_s8(shift_vals); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + const int8x16_t one = vdupq_n_s8(1); + + float32x4_t sumv = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + + // Load 8 bytes of packed 2-bit values + const uint8x8_t raw = vld1_u8(&x[i].qs[k * 8]); + const uint8x16_t raw16 = vcombine_u8(raw, raw); + + // First 16 elements: replicate bytes 0-3, shift, mask, subtract 1 + uint8x16_t bytes0 = vqtbl1q_u8(raw16, idx_lo); + int8x16_t qv0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)), + one); + + // Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1 + uint8x16_t bytes1 = vqtbl1q_u8(raw16, idx_hi); + int8x16_t qv1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)), + one); + + // Load Q8_0 values and dot product + const int8x16_t y0 = vld1q_s8(yb->qs); + const int8x16_t y1 = vld1q_s8(yb->qs + 16); + + int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), qv0, y0); + int32x4_t p1 = ggml_vdotq_s32(p0, qv1, y1); + + sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1); + } + } + + sumf = vaddvq_f32(sumv); +#else + ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + return; +#endif + + *s = sumf; +} void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 2b3eb5b5ce65..746cef2cdd3f 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -223,6 +223,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_Q2_0] = { + .from_float = quantize_row_q2_0, + .vec_dot = ggml_vec_dot_q2_0_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q4_0] = { .from_float = quantize_row_q4_0, .vec_dot = ggml_vec_dot_q4_0_q8_0, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index a9bc21da6f0f..6c31ac1aebbb 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -665,6 +665,7 @@ void ggml_compute_forward_add( ggml_compute_forward_add_non_quantized(params, dst); } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1115,6 +1116,7 @@ void ggml_compute_forward_add1( } } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1245,6 +1247,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4335,6 +4338,7 @@ void ggml_compute_forward_out_prod( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4611,6 +4615,7 @@ void ggml_compute_forward_set( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4835,6 +4840,7 @@ void ggml_compute_forward_get_rows( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5561,6 +5567,7 @@ void ggml_compute_forward_clamp( } break; case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index e5f9a4083f9c..f92f554f443f 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -26,6 +26,10 @@ void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in quantize_row_q1_0_ref(x, y, k); } +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_q2_0_ref(x, y, k); +} + void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_q4_0_ref(x, y, k); } @@ -170,6 +174,52 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + float sumi = 0.0f; + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + + const uint8_t * GGML_RESTRICT qs = &x[i].qs[k * 8]; + const int8_t * GGML_RESTRICT qy = yb->qs; + + for (int b = 0; b < 8; ++b) { + const uint8_t byte = qs[b]; + // Extract 4 two-bit values, map {0,1,2,3} -> {-1,0,1,2} + sumi_block += ((int)((byte >> 0) & 3) - 1) * qy[b*4 + 0]; + sumi_block += ((int)((byte >> 2) & 3) - 1) * qy[b*4 + 1]; + sumi_block += ((int)((byte >> 4) & 3) - 1) * qy[b*4 + 2]; + sumi_block += ((int)((byte >> 6) & 3) - 1) * qy[b*4 + 3]; + } + + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +} void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index d4bc87a1c052..93ea7eeffe5b 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -13,6 +13,7 @@ extern "C" { // Quantization void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -38,6 +39,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, // Dot product void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -71,6 +73,7 @@ void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRI void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 15443aa554a4..b2454564d648 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -67,6 +67,44 @@ void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_REST } } +void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + // Compute scale as max absolute value in the block + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float a = fabsf(x[i*qk + j]); + if (a > amax) amax = a; + } + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + // Clear quant bytes + for (int j = 0; j < qk / 4; ++j) { + y[i].qs[j] = 0; + } + + // Encode 2-bit values: round(w/d) clamped to [-1, 2], then add 1 + // 00 (-1) = -scale, 01 (0) = 0, 10 (+1) = +scale, 11 (+2) = 2*scale + for (int j = 0; j < qk; ++j) { + const float w = x[i*qk + j]; + int q = (int)roundf(w * id) + 1; + if (q < 0) q = 0; + if (q > 3) q = 3; + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + y[i].qs[byte_index] |= ((uint8_t)q << bit_offset); + } + } +} + // reference implementation for deterministic creation of model files void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -394,6 +432,26 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI } } +void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(x[i].d); + + for (int j = 0; j < qk; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const uint8_t q = (x[i].qs[byte_index] >> bit_offset) & 0x03; + // 00=-1, 01=0, 10=+1, 11=+2 + y[i*qk + j] = ((int)q - 1) * d; + } + } +} + void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -2048,6 +2106,20 @@ size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } +size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + if (!quant_weights) { + quantize_row_q2_0_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + } + size_t row_size = ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_q2_0_ref(src, (block_q2_0*)qrow, n_per_row); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { if (!quant_weights) { @@ -5361,6 +5433,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb); } break; + case GGML_TYPE_Q2_0: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb); + } break; case GGML_TYPE_Q4_0: { VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index d56c86da8909..75188f1af180 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -15,6 +15,7 @@ extern "C" { // Quantization GGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k); @@ -43,6 +44,7 @@ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_ // Dequantization GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -93,6 +95,7 @@ GGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index eda041f4518a..6ea85acd0d99 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -669,6 +669,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_q1_0, .from_float_ref = (ggml_from_float_t) quantize_row_q1_0_ref, }, + [GGML_TYPE_Q2_0] = { + .type_name = "q2_0", + .blck_size = QK2_0, + .type_size = sizeof(block_q2_0), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q2_0, + .from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref, + }, [GGML_TYPE_Q4_0] = { .type_name = "q4_0", .blck_size = QK4_0, @@ -1403,6 +1411,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break; case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break; case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break; + case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break; case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break; case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break; case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break; @@ -7674,6 +7683,7 @@ size_t ggml_quantize_chunk( switch (type) { case GGML_TYPE_Q1_0: result = quantize_q1_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q2_0: result = quantize_q2_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_0: result = quantize_q4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_1: result = quantize_q4_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q5_0: result = quantize_q5_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c5297a2f440f..ac1f767d484c 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4006,6 +4006,7 @@ class GGMLQuantizationType(IntEnum): MXFP4 = 39 NVFP4 = 40 Q1_0 = 41 + Q2_0 = 42 class ExpertGatingFuncType(IntEnum): @@ -4060,6 +4061,7 @@ class LlamaFileType(IntEnum): MOSTLY_MXFP4_MOE = 38 # except 1d tensors MOSTLY_NVFP4 = 39 # except 1d tensors MOSTLY_Q1_0 = 40 # except 1d tensors + MOSTLY_Q2_0 = 41 # except 1d tensors GUESSED = 1024 # not specified in the model file @@ -4177,6 +4179,7 @@ class VisionProjectorType: GGMLQuantizationType.MXFP4: (32, 1 + 16), GGMLQuantizationType.NVFP4: (64, 4 + 32), GGMLQuantizationType.Q1_0: (128, 2 + 16), + GGMLQuantizationType.Q2_0: (128, 2 + 32), } diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py index 1d9d9ab7d70e..5a8e968ac11f 100644 --- a/gguf-py/gguf/quants.py +++ b/gguf-py/gguf/quants.py @@ -653,6 +653,45 @@ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: return (d * qs.astype(np.float32)) +class Q2_0(__Quant, qtype=GGMLQuantizationType.Q2_0): + @classmethod + def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + n_blocks = blocks.shape[0] + + # Compute scale as max absolute value per block + d = np.abs(blocks).max(axis=-1, keepdims=True) + + with np.errstate(divide="ignore"): + id = np.where(d == 0, 0, 1 / d) + qs = np_roundf(blocks * id) + qs = np.clip(qs, -1, 2).astype(np.int8) + np.int8(1) + qs = qs.astype(np.uint8) + + # Pack 4 values per byte: [v0:1:0, v1:3:2, v2:5:4, v3:7:6] + qs = qs.reshape((n_blocks, -1, 4)) << np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4)) + qs = qs[..., 0] | qs[..., 1] | qs[..., 2] | qs[..., 3] + qs = qs.reshape((n_blocks, -1)) + + d = d.astype(np.float16).view(np.uint8) + + # Layout: [d (2 bytes), qs (32 bytes)] + return np.concatenate([d, qs], axis=-1) + + @classmethod + def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + n_blocks = blocks.shape[0] + + d, qs = np.hsplit(blocks, [2]) + + d = d.view(np.float16).astype(np.float32) + + # Unpack 4 values per byte + qs = qs.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4)) + qs = (qs & 0x03).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1) + + return (d * qs.astype(np.float32)) + + class MXFP4(__Quant, qtype=GGMLQuantizationType.MXFP4): # e2m1 values (doubled) # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf diff --git a/include/llama.h b/include/llama.h index ac267b5089ab..8cdc5aca4bfa 100644 --- a/include/llama.h +++ b/include/llama.h @@ -155,6 +155,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 4e65a45a50d8..65d7b2110742 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -37,6 +37,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_F16: return "F16"; case LLAMA_FTYPE_MOSTLY_BF16: return "BF16"; case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0"; + case LLAMA_FTYPE_MOSTLY_Q2_0: return "Q2_0"; case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0"; case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1"; case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0"; @@ -760,6 +761,7 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; + case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index f91d795b3e96..57e5bd917da8 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -380,6 +380,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: // types on the right: block size 32 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break; + case GGML_TYPE_Q2_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_TQ1_0: @@ -480,7 +481,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_IQ3_S; } - else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0) { + else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) { new_type = GGML_TYPE_Q4_K; } } @@ -800,6 +801,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16; case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32; case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0; + case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0; case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4; diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 3d33d47d98b5..fce5f2ed2033 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -33,6 +33,7 @@ struct quant_option { static const std::vector QUANT_OPTIONS = { { "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", }, + { "Q2_0", LLAMA_FTYPE_MOSTLY_Q2_0, " 2.125 bpw quantization", }, { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", }, { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", }, { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", }, From 0eed534000c65eea5937fef39cf12d62297db548 Mon Sep 17 00:00:00 2001 From: Pasha Khosravi Date: Fri, 17 Apr 2026 21:34:09 -0700 Subject: [PATCH 04/11] Add Q2_0 Metal backend --- ggml/src/ggml-metal/ggml-metal-device.cpp | 10 ++ ggml/src/ggml-metal/ggml-metal-device.m | 2 + ggml/src/ggml-metal/ggml-metal-impl.h | 3 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 1 + ggml/src/ggml-metal/ggml-metal.metal | 201 ++++++++++++++++++++++ 5 files changed, 217 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 07d016d22277..8deb5136b6ee 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -742,6 +742,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; @@ -959,6 +964,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 27cb16835187..006b2f53e66e 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1204,6 +1204,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_TYPE_BF16: case GGML_TYPE_Q8_0: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1231,6 +1232,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return false; } case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 379a8b33a143..99bb80b77c4c 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -11,6 +11,9 @@ #define N_R0_Q1_0 8 #define N_SG_Q1_0 2 +#define N_R0_Q2_0 8 +#define N_SG_Q2_0 2 + #define N_R0_Q4_0 4 #define N_SG_Q4_0 2 diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index e173527909ad..b3979eed1c73 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2059,6 +2059,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16 || op->src[0]->type == GGML_TYPE_Q1_0 || + op->src[0]->type == GGML_TYPE_Q2_0 || op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_0 || diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 9f38c9d2968f..87b1cf6eb8f7 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -168,6 +168,39 @@ void dequantize_q1_0_t4(device const block_q1_0 * xb, short il, thread type4 & r reg = (type4) reg_f; } +template +void dequantize_q2_0(device const block_q2_0 * xb, short il, thread type4x4 & reg) { + device const uint8_t * qs = xb->qs; + const float d = xb->d; + + const int byte_offset = il * 4; // il*16 elements = il*4 bytes (4 elements per byte) + float4x4 reg_f; + + for (int i = 0; i < 4; i++) { + const uint8_t b = qs[byte_offset + i]; + reg_f[i][0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[i][1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[i][2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[i][3] = ((float)((b >> 6) & 3) - 1.0f) * d; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q2_0_t4(device const block_q2_0 * xb, short il, thread type4 & reg) { + const float d = xb->d; + const uint8_t b = xb->qs[il]; + + float4 reg_f; + reg_f[0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[3] = ((float)((b >> 6) & 3) - 1.0f) * d; + + reg = (type4) reg_f; +} + template void dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & reg) { device const uint16_t * qs = ((device const uint16_t *)xb + 1); @@ -219,6 +252,27 @@ void quantize_q1_0(device const float * src, device block_q1_0 & dst) { } } +void quantize_q2_0(device const float * src, device block_q2_0 & dst) { + float amax = 0.0f; + for (int j = 0; j < QK2_0; j++) { + float a = fabs(src[j]); + if (a > amax) amax = a; + } + const float d = amax; + dst.d = d; + + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + for (int j = 0; j < QK2_0 / 4; j++) { + dst.qs[j] = 0; + } + for (int j = 0; j < QK2_0; j++) { + int q = (int)round(src[j] * id) + 1; + q = max(0, min(3, q)); + dst.qs[j / 4] |= (q << (2 * (j % 4))); + } +} + void quantize_q4_0(device const float * src, device block_q4_0 & dst) { #pragma METAL fp math_mode(safe) float amax = 0.0f; // absolute max @@ -3221,6 +3275,60 @@ inline float block_q_n_dot_y(device const block_q1_0 * qb_curr, float sumy, thre return qb_curr->d * (2.0f * acc - sumy); } +// Q2_0 dot product: dot = d * (Σ(q_raw[i] * yl[i]) - sumy) +// q_raw are unsigned 2-bit values {0,1,2,3}, mapping: value = (q_raw - 1) * d +// Q2_0 dot product using bit-decomposition: +// value = (low_bit + 2*high_bit - 1) +// sum(value * y) = sum_lo(y) + 2*sum_hi(y) - sumy +// where sum_lo/sum_hi use Q1_0-style conditional adds (no multiplies) +inline float block_q_n_dot_y(device const block_q2_0 * qb_curr, float sumy, thread float * yl, int il) { + device const uint8_t * qs = qb_curr->qs + (il / 4); + const uint8_t b0 = qs[0]; + const uint8_t b1 = qs[1]; + const uint8_t b2 = qs[2]; + const uint8_t b3 = qs[3]; + + // Accumulate where low bit is set (bits 0,2,4,6 of each byte) + float acc_lo = 0.0f; + acc_lo += select(0.0f, yl[ 0], bool(b0 & 0x01)); + acc_lo += select(0.0f, yl[ 1], bool(b0 & 0x04)); + acc_lo += select(0.0f, yl[ 2], bool(b0 & 0x10)); + acc_lo += select(0.0f, yl[ 3], bool(b0 & 0x40)); + acc_lo += select(0.0f, yl[ 4], bool(b1 & 0x01)); + acc_lo += select(0.0f, yl[ 5], bool(b1 & 0x04)); + acc_lo += select(0.0f, yl[ 6], bool(b1 & 0x10)); + acc_lo += select(0.0f, yl[ 7], bool(b1 & 0x40)); + acc_lo += select(0.0f, yl[ 8], bool(b2 & 0x01)); + acc_lo += select(0.0f, yl[ 9], bool(b2 & 0x04)); + acc_lo += select(0.0f, yl[10], bool(b2 & 0x10)); + acc_lo += select(0.0f, yl[11], bool(b2 & 0x40)); + acc_lo += select(0.0f, yl[12], bool(b3 & 0x01)); + acc_lo += select(0.0f, yl[13], bool(b3 & 0x04)); + acc_lo += select(0.0f, yl[14], bool(b3 & 0x10)); + acc_lo += select(0.0f, yl[15], bool(b3 & 0x40)); + + // Accumulate where high bit is set (bits 1,3,5,7 of each byte) + float acc_hi = 0.0f; + acc_hi += select(0.0f, yl[ 0], bool(b0 & 0x02)); + acc_hi += select(0.0f, yl[ 1], bool(b0 & 0x08)); + acc_hi += select(0.0f, yl[ 2], bool(b0 & 0x20)); + acc_hi += select(0.0f, yl[ 3], bool(b0 & 0x80)); + acc_hi += select(0.0f, yl[ 4], bool(b1 & 0x02)); + acc_hi += select(0.0f, yl[ 5], bool(b1 & 0x08)); + acc_hi += select(0.0f, yl[ 6], bool(b1 & 0x20)); + acc_hi += select(0.0f, yl[ 7], bool(b1 & 0x80)); + acc_hi += select(0.0f, yl[ 8], bool(b2 & 0x02)); + acc_hi += select(0.0f, yl[ 9], bool(b2 & 0x08)); + acc_hi += select(0.0f, yl[10], bool(b2 & 0x20)); + acc_hi += select(0.0f, yl[11], bool(b2 & 0x80)); + acc_hi += select(0.0f, yl[12], bool(b3 & 0x02)); + acc_hi += select(0.0f, yl[13], bool(b3 & 0x08)); + acc_hi += select(0.0f, yl[14], bool(b3 & 0x20)); + acc_hi += select(0.0f, yl[15], bool(b3 & 0x80)); + + return qb_curr->d * (acc_lo + 2.0f * acc_hi - sumy); +} + // function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i]) // il indicates where the q4 quants begin (0 or QK4_0/4) // we assume that the yl's have been multiplied with the appropriate scale factor @@ -3521,6 +3629,85 @@ kernel void kernel_mul_mv_q1_0_f32( kernel_mul_mv_q1_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } +template +void kernel_mul_mv_q2_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK2_0; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%args.ne12; + const uint i13 = im/args.ne12; + + const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13; + + device const float * y = (device const float *) (src1 + offset1); + + device const block_q2_0 * ax[nr0]; + for (int row = 0; row < nr0; ++row) { + const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; + ax[row] = (device const block_q2_0 *) ((device char *) src0 + offset0); + } + + float yl[16]; + float sumf[nr0] = {0.f}; + + const short ix = (tiisg/8); + const short il = (tiisg%8)*16; + + device const float * yb = y + ix*QK2_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + float sumy = 0.f; + + FOR_UNROLL (short i = 0; i < 16; i++) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + FOR_UNROLL (short row = 0; row < nr0; row++) { + sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += QK2_0 * (N_SIMDWIDTH/8); + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0; ++row) { + const float tot = simd_sum(sumf[row]); + + if (tiisg == 0 && first_row + row < args.ne01) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_q2_0_f32")]] +kernel void kernel_mul_mv_q2_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_q2_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + kernel void kernel_mul_mv_q4_0_f32( constant ggml_metal_kargs_mul_mv & args, device const char * src0, @@ -3918,6 +4105,11 @@ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_3")]] kernel mul_mv_ext_q4 template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q1_0, 128, dequantize_q1_0_t4>; template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q1_0, 128, dequantize_q1_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q2_0, 128, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q2_0, 128, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q2_0, 128, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q2_0, 128, dequantize_q2_0_t4>; + template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>; template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>; template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>; @@ -7357,6 +7549,7 @@ typedef decltype(kernel_cpy_f32_q) cpy_f_q_ template [[host_name("kernel_cpy_f32_q8_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q1_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q4_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; @@ -7398,6 +7591,7 @@ kernel void kernel_cpy_q_f32( typedef decltype(kernel_cpy_q_f32) cpy_q_f_t; template [[host_name("kernel_cpy_q1_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; @@ -7405,6 +7599,7 @@ template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32< template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q4_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; @@ -10003,6 +10198,7 @@ template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_ro typedef decltype(kernel_get_rows_q) get_rows_q_t; template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q2_0")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; @@ -10066,6 +10262,7 @@ template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; #endif template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mul_mm_t kernel_mul_mm; @@ -10090,6 +10287,7 @@ template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q1_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_1_f16")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q5_0_f16")]] kernel mul_mm_t kernel_mul_mm; @@ -10123,6 +10321,7 @@ template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mul_mm_id kernel_m template [[host_name("kernel_mul_mm_id_bf16_f32")]] kernel mul_mm_id kernel_mul_mm_id; #endif template [[host_name("kernel_mul_mm_id_q1_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; @@ -10147,6 +10346,7 @@ template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_m template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q1_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q4_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q5_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; @@ -10302,6 +10502,7 @@ template [[host_name("kernel_mul_mv_id_bf16_f32_4")]] kernel kernel_mul_mv_id_4 template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; From e380897eccb924f830d75a7ed9eeb81a07bd7f20 Mon Sep 17 00:00:00 2001 From: Pasha Khosravi Date: Sat, 18 Apr 2026 08:00:18 +0000 Subject: [PATCH 05/11] cuda: Q2_0 --- ggml/src/ggml-cpu/arch-fallback.h | 7 ++ ggml/src/ggml-cuda/common.cuh | 7 ++ ggml/src/ggml-cuda/convert.cu | 10 ++ ggml/src/ggml-cuda/dequantize.cuh | 20 ++++ ggml/src/ggml-cuda/getrows.cu | 4 + ggml/src/ggml-cuda/ggml-cuda.cu | 2 + ggml/src/ggml-cuda/mmq.cu | 4 + ggml/src/ggml-cuda/mmq.cuh | 106 ++++++++++++++++++ ggml/src/ggml-cuda/mmvq.cu | 8 ++ .../template-instances/generate_cu_files.py | 2 +- .../template-instances/mmq-instance-q2_0.cu | 5 + ggml/src/ggml-cuda/vecdotq.cuh | 61 ++++++++++ 12 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 595ded09f03f..1758d83c261a 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -17,6 +17,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -83,6 +84,7 @@ #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 @@ -114,6 +116,7 @@ #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K @@ -163,6 +166,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -204,6 +208,7 @@ // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 @@ -245,6 +250,7 @@ #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -309,6 +315,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index ddf50baf4956..5a5c7eaeef39 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -931,6 +931,13 @@ struct ggml_cuda_type_traits { static constexpr int qi = QI1_0; }; +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK2_0; + static constexpr int qr = QR2_0; + static constexpr int qi = QI2_0; +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK4_0; diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 61630a35a29b..3f121842f5d6 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -713,6 +713,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -771,6 +773,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -828,6 +832,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -851,6 +857,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -874,6 +882,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuh index 9ae1342fc0ef..f5490a440823 100644 --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -22,6 +22,26 @@ static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const in v.y = (2*bit_1 - 1) * d; } +static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_q2_0 * x = (const block_q2_0 *) vx; + + const float d = x[ib].d; + + // Q2_0: 2 bits per element, 4 elements per byte. + // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}. + const int byte_index_0 = iqs / 4; + const int bit_offset_0 = (iqs % 4) * 2; + + const int byte_index_1 = (iqs + 1) / 4; + const int bit_offset_1 = ((iqs + 1) % 4) * 2; + + const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3; + const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3; + + v.x = (c0 - 1) * d; + v.y = (c1 - 1) * d; +} + static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index e99cba63d344..d0882c739d50 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -183,6 +183,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_Q2_0: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q4_0: get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index de579d2ed508..c7cbeb886940 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4841,6 +4841,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4879,6 +4880,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_BF16: case GGML_TYPE_I32: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 3f01ff5bfb08..152d61ae1ec7 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -8,6 +8,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con case GGML_TYPE_Q1_0: mul_mat_q_case(ctx, args, stream); break; + case GGML_TYPE_Q2_0: + mul_mat_q_case(ctx, args, stream); + break; case GGML_TYPE_Q4_0: mul_mat_q_case(ctx, args, stream); break; @@ -274,6 +277,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 28b662df9258..c04d9e4cfcb3 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -58,6 +58,7 @@ static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected b static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { switch (type_x) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: return MMQ_Q8_1_DS_LAYOUT_D4; case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -188,6 +189,7 @@ static constexpr __device__ int get_mmq_y_device() { static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int mmq_y) { switch (type) { case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0; case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1; case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0; @@ -233,6 +235,7 @@ static_assert(MMQ_MMA_TILE_X_K_NVFP4 % 8 == 4, "Wrong padding."); static constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return MMQ_MMA_TILE_X_K_Q8_0; + case GGML_TYPE_Q2_0: return MMQ_MMA_TILE_X_K_Q8_0; case GGML_TYPE_Q4_0: return MMQ_MMA_TILE_X_K_Q8_0; case GGML_TYPE_Q4_1: return MMQ_MMA_TILE_X_K_Q8_1; case GGML_TYPE_Q5_0: return MMQ_MMA_TILE_X_K_Q8_0; @@ -387,6 +390,101 @@ template static __device__ __forceinline__ void loa } } +template static __device__ __forceinline__ void load_tiles_q2_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int nwarps = mmq_get_nwarps_device(); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0; + constexpr int threads_per_row = blocks_per_iter * QI2_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK2_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI2_0; + const int kqsx = txi % QI2_0; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (need_check) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx; + // Each 32-element chunk occupies 8 bytes of qs (32 elements * 2 bits = 64 bits) + const int qs_offset = 8*kqsx; + const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) | + (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24); + const int qs1 = bxi->qs[qs_offset + 4] | (bxi->qs[qs_offset + 5] << 8) | + (bxi->qs[qs_offset + 6] << 16) | (bxi->qs[qs_offset + 7] << 24); + + // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8s in {-1,0,1,2}. + int unpacked_bytes[8]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int shift = j * 8; + const int codes = (qs0 >> shift) & 0xFF; + const int c0 = ((codes >> 0) & 0x3) - 1; + const int c1 = ((codes >> 2) & 0x3) - 1; + const int c2 = ((codes >> 4) & 0x3) - 1; + const int c3 = ((codes >> 6) & 0x3) - 1; + unpacked_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int shift = j * 8; + const int codes = (qs1 >> shift) & 0xFF; + const int c0 = ((codes >> 0) & 0x3) - 1; + const int c1 = ((codes >> 2) & 0x3) - 1; + const int c2 = ((codes >> 4) & 0x3) - 1; + const int c3 = ((codes >> 6) & 0x3) - 1; + unpacked_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); + } + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; +#pragma unroll + for (int j = 0; j < 8; ++j) { +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + dst_offset + j] = unpacked_bytes[j]; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j]; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + template static __device__ __forceinline__ void load_tiles_q4_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int nwarps = mmq_get_nwarps_device(); @@ -3383,6 +3481,14 @@ struct mmq_type_traits { static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; }; +template +struct mmq_type_traits { + static constexpr int vdr = VDR_Q2_0_Q8_1_MMQ; + static constexpr load_tiles_mmq_t load_tiles = load_tiles_q2_0; + static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; + static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; +}; + template struct mmq_type_traits { static constexpr int vdr = VDR_Q4_0_Q8_1_MMQ; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 8f55cace1a1e..d096b0833229 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -10,6 +10,7 @@ typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1; + case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1; case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1; case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1; case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1; @@ -38,6 +39,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ; + case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ; case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ; case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ; case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ; @@ -894,6 +896,12 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q2_0: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_Q4_0: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 841059c15b52..717871002d7a 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -32,7 +32,7 @@ SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size_kq}, {head_size_v}, {ncols1}, {ncols2});\n" TYPES_MMQ = [ - "GGML_TYPE_Q1_0", + "GGML_TYPE_Q1_0", "GGML_TYPE_Q2_0", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K", "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S", diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu new file mode 100644 index 000000000000..750180e3306d --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_Q2_0); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index d1741cc8d7ba..b88393acf48e 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -109,6 +109,9 @@ static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) { #define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism #define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block +#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism +#define VDR_Q2_0_Q8_1_MMQ 4 // Q2_0 has 256 bits (8 ints) per block, 4 32-element chunks + #define VDR_Q4_0_Q8_1_MMVQ 2 #define VDR_Q4_0_Q8_1_MMQ 4 @@ -717,6 +720,64 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1( return d1 * d8 * sumi; } +static __device__ __forceinline__ float vec_dot_q2_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx; + + // Q2_0: 128 elements with ONE scale, 2 bits per element (4 elements per byte) + // Q8_1: 32 elements per block with individual scales + // iqs selects which of the 4 chunks of 32 elements to process (0-3) + + const float d2 = bq2_0->d; + + // Process only the chunk specified by iqs + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + + // Load 64 bits (8 bytes) for this chunk from Q2_0: bytes [8*iqs, 8*iqs+8) + const int offset = iqs * 8; + const int v0 = bq2_0->qs[offset + 0] | (bq2_0->qs[offset + 1] << 8) | + (bq2_0->qs[offset + 2] << 16) | (bq2_0->qs[offset + 3] << 24); + const int v1 = bq2_0->qs[offset + 4] | (bq2_0->qs[offset + 5] << 8) | + (bq2_0->qs[offset + 6] << 16) | (bq2_0->qs[offset + 7] << 24); + + // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8 symbols in {-1,0,1,2}. + // Stored code c in {0,1,2,3} -> symbol s = c - 1. + int vi_bytes[8]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int shift = j * 8; + const int codes = (v0 >> shift) & 0xFF; + const int c0 = ((codes >> 0) & 0x3) - 1; + const int c1 = ((codes >> 2) & 0x3) - 1; + const int c2 = ((codes >> 4) & 0x3) - 1; + const int c3 = ((codes >> 6) & 0x3) - 1; + vi_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int shift = j * 8; + const int codes = (v1 >> shift) & 0xFF; + const int c0 = ((codes >> 0) & 0x3) - 1; + const int c1 = ((codes >> 2) & 0x3) - 1; + const int c2 = ((codes >> 4) & 0x3) - 1; + const int c3 = ((codes >> 6) & 0x3) - 1; + vi_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); + } + + // Compute dot product for this 32-element chunk + int sumi = 0; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int u = get_int_b4(bq8_1_chunk->qs, j); + sumi = ggml_cuda_dp4a(vi_bytes[j], u, sumi); + } + + // Apply Q2_0's single scale and this chunk's Q8_1 scale + const float d8 = __low2float(bq8_1_chunk->ds); + return d2 * d8 * sumi; +} + static __device__ __forceinline__ float vec_dot_q4_0_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { From d104cf1b639a909ddea521d61f7cb023c6e41f57 Mon Sep 17 00:00:00 2001 From: Pasha Khosravi Date: Sun, 19 Apr 2026 00:50:55 -0700 Subject: [PATCH 06/11] release-prism: install spirv-headers for ubuntu-arm64 vulkan build --- .github/workflows/release-prism.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-prism.yml b/.github/workflows/release-prism.yml index cc9fb008f7cd..923903c83026 100644 --- a/.github/workflows/release-prism.yml +++ b/.github/workflows/release-prism.yml @@ -345,7 +345,7 @@ jobs: sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev else sudo apt-get update -y - sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev libssl-dev ninja-build + sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build echo "CC=gcc-14" >> "$GITHUB_ENV" echo "CXX=g++-14" >> "$GITHUB_ENV" fi From d11c45d59dc433d4045baeb1b0e6be01c63181aa Mon Sep 17 00:00:00 2001 From: pl752 Date: Sat, 2 May 2026 03:09:06 +0500 Subject: [PATCH 07/11] Implemented bit-interleaved Q1_0 8x32 repack kernels for x86 AVX2 --- ggml/src/ggml-cpu/arch-fallback.h | 22 ++ ggml/src/ggml-cpu/arch/x86/repack.cpp | 304 ++++++++++++++++++++++++++ ggml/src/ggml-cpu/repack.cpp | 238 ++++++++++++++++++++ ggml/src/ggml-cpu/repack.h | 11 + 4 files changed, 575 insertions(+) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 1758d83c261a..1e3a4521c5b8 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -39,6 +39,7 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -55,6 +56,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -71,16 +73,20 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -125,6 +131,7 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -141,6 +148,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -157,6 +165,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #elif defined(__loongarch64) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -172,6 +181,7 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -188,6 +198,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -204,12 +215,14 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1 @@ -230,6 +243,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -245,6 +259,8 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 + #elif defined(__s390x__) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -266,6 +282,7 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -282,6 +299,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -298,6 +316,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #elif defined(__wasm__) // quants.c #define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1 @@ -321,6 +340,7 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -337,6 +357,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -353,4 +374,5 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 #endif diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index af1cebad131d..d97c90c21c7a 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -287,6 +287,75 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR #endif } +void ggml_quantize_mat_q8_0_4x32(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + +#if defined(__AVX2__) + for (int i = 0; i < nb; i++) { + for (int r = 0; r < 4; r++) { + // Load elements into 4 AVX vectors + __m256 v0 = _mm256_loadu_ps( x + r * k + i * 32 ); + __m256 v1 = _mm256_loadu_ps( x + r * k + i * 32 + 8 ); + __m256 v2 = _mm256_loadu_ps( x + r * k + i * 32 + 16 ); + __m256 v3 = _mm256_loadu_ps( x + r * k + i * 32 + 24 ); + + // Compute max(abs(e)) for the block + const __m256 signBit = _mm256_set1_ps( -0.0f ); + __m256 maxAbs = _mm256_andnot_ps( signBit, v0 ); + maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) ); + maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) ); + maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) ); + + __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); + max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); + max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); + const float maxScalar = _mm_cvtss_f32( max4 ); + + const float d = maxScalar / 127.f; + y[i].d[r] = GGML_CPU_FP32_TO_FP16(d); + const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; + const __m256 mul = _mm256_set1_ps( id ); + + // Apply multiplier + v0 = _mm256_mul_ps( v0, mul ); + v1 = _mm256_mul_ps( v1, mul ); + v2 = _mm256_mul_ps( v2, mul ); + v3 = _mm256_mul_ps( v3, mul ); + + // Round + v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); + v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); + v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); + v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); + + // Convert to int + __m256i i0 = _mm256_cvtps_epi32( v0 ); + __m256i i1 = _mm256_cvtps_epi32( v1 ); + __m256i i2 = _mm256_cvtps_epi32( v2 ); + __m256i i3 = _mm256_cvtps_epi32( v3 ); + + i0 = _mm256_packs_epi32( i0, i1 ); + i2 = _mm256_packs_epi32( i2, i3 ); + i0 = _mm256_packs_epi16( i0, i2 ); + + const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); + i0 = _mm256_permutevar8x32_epi32( i0, perm ); + + // Store row r contiguously + _mm256_storeu_si256((__m256i *)(y[i].qs + r * 32), i0); + } + } +#else + UNUSED(nb); + UNUSED(y); + ggml_quantize_mat_q8_0_4x32_generic(x, vy, k); +#endif +} + void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK_K == 256); assert(k % QK_K == 0); @@ -1461,6 +1530,147 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } +void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined( __AVX2__ ) || defined( __AVX512F__ ) + { + assert (n % QK1_0 == 0); + assert (nc % 8 == 0); + + UNUSED(bs); + UNUSED(nr); + + const int nb = n / QK1_0; + const int nb32 = n / QK8_0; + const int ncols8 = nc / 8; + + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i zero = _mm256_setzero_si256(); + + // Shuffle LUTs for columns 0-3: LUT[b & 0xF] = (b >> c) & 1 ? 0x00 : 0xFF + alignas(32) static const uint8_t sm_lut_c0[16] = { + 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, + 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00 + }; + alignas(32) static const uint8_t sm_lut_c1[16] = { + 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, + 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00 + }; + alignas(32) static const uint8_t sm_lut_c2[16] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00 + }; + alignas(32) static const uint8_t sm_lut_c3[16] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + + const __m256i lut[4] = { + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c0)), + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c1)), + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c2)), + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c3)), + }; + + // Column masks for columns 4-7 (AND+cmpeq path) + const __m256i col_mask_4 = _mm256_set1_epi8(16); + const __m256i col_mask_5 = _mm256_set1_epi8(32); + const __m256i col_mask_6 = _mm256_set1_epi8(64); + const __m256i col_mask_7 = _mm256_set1_epi8((int8_t)-128); + + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + const block_q8_0 * a_ptr = (const block_q8_0 *)vy; + + for (int y = 0; y < nr; ++y) { + const block_q8_0 * a_row = a_ptr + (size_t)y * nb32; + float * row_out = s + (size_t)y * nc; + + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + + __m256 acc[8]; + for (int c = 0; c < 8; ++c) acc[c] = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + + __m256 block_acc[8]; + for (int c = 0; c < 8; ++c) block_acc[c] = _mm256_setzero_ps(); + + const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * yb = &a_row[l * 4 + sb]; + const __m256i rhs = _mm256_loadu_si256((const __m256i *)yb->qs); + const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + + const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); + + // Columns 0-3: shuffle LUT on low 7 bits + const __m256i qs_lo7 = _mm256_and_si256(qs_vec, _mm256_set1_epi8(0x7F)); + const __m256i sm0 = _mm256_shuffle_epi8(lut[0], qs_lo7); + const __m256i sm1 = _mm256_shuffle_epi8(lut[1], qs_lo7); + const __m256i sm2 = _mm256_shuffle_epi8(lut[2], qs_lo7); + const __m256i sm3 = _mm256_shuffle_epi8(lut[3], qs_lo7); + + // Columns 4-7: AND + cmpeq + const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_4), zero); + const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_5), zero); + const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_6), zero); + const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_7), zero); + + // Sign-flip and accumulate for all 8 columns + const __m256i sy0 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm0), sm0); + const __m256i sy1 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm1), sm1); + const __m256i sy2 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm2), sm2); + const __m256i sy3 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm3), sm3); + const __m256i sy4 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm4), sm4); + const __m256i sy5 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm5), sm5); + const __m256i sy6 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm6), sm6); + const __m256i sy7 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm7), sm7); + + const __m256i s32_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16); + const __m256i s32_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16); + const __m256i s32_2 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16); + const __m256i s32_3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16); + const __m256i s32_4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16); + const __m256i s32_5 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16); + const __m256i s32_6 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16); + const __m256i s32_7 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16); + + block_acc[0] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_0), block_acc[0]); + block_acc[1] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_1), block_acc[1]); + block_acc[2] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_2), block_acc[2]); + block_acc[3] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_3), block_acc[3]); + block_acc[4] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_4), block_acc[4]); + block_acc[5] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_5), block_acc[5]); + block_acc[6] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_6), block_acc[6]); + block_acc[7] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_7), block_acc[7]); + } + + for (int c = 0; c < 8; ++c) { + acc[c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[c], acc[c]); + } + } + + // Reduce 8 lanes to 1 value per column and store + for (int c = 0; c < 8; ++c) { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc[c]), _mm256_extractf128_ps(acc[c], 1)); + const __m128 t = _mm_hadd_ps(v, v); + row_out[x * 8 + c] = _mm_cvtss_f32(_mm_hadd_ps(t, t)); + } + } + } + + return; + } +#endif // defined( __AVX2__ ) || defined( __AVX512F__ ) + + ggml_gemv_q1_0_8x32_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK_K; const int nb = n / qk; @@ -2039,6 +2249,100 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } +void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined( __AVX2__ ) || defined( __AVX512F__ ) + { + assert (n % QK1_0 == 0); + assert (nr % 4 == 0); + assert (nc % 8 == 0); + + UNUSED(bs); + + const int nb = n / QK1_0; + const int nb_q8_0 = n / QK8_0; + const int ncols8 = nc / 8; + const int nrows4 = nr / 4; + + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i zero = _mm256_setzero_si256(); + + const uint8_t col_masks[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + + for (int y = 0; y < nrows4; ++y) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *)vy + (y * nb_q8_0); + + for (int row_base = 0; row_base < 4; row_base += 2) { + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + + __m256 acc[2][8]; + for (int r = 0; r < 2; ++r) + for (int c = 0; c < 8; ++c) + acc[r][c] = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + + __m256 block_acc[2][8]; + for (int r = 0; r < 2; ++r) + for (int c = 0; c < 8; ++c) + block_acc[r][c] = _mm256_setzero_ps(); + + const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; + const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); + + __m256i sm[8]; + for (int c = 0; c < 8; ++c) { + const __m256i mask_c = _mm256_set1_epi8((int8_t)col_masks[c]); + sm[c] = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, mask_c), zero); + } + + for (int r = 0; r < 2; ++r) { + const __m256i rhs = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + r) * 32)); + const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + r])); + + for (int c = 0; c < 8; ++c) { + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm[c]), sm[c]); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + block_acc[r][c] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32), block_acc[r][c]); + } + } + } + + for (int r = 0; r < 2; ++r) { + for (int c = 0; c < 8; ++c) { + acc[r][c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[r][c], acc[r][c]); + } + } + } + + float * s_row0 = s + (y * 4 + row_base + 0) * bs + x * 8; + float * s_row1 = s + (y * 4 + row_base + 1) * bs + x * 8; + for (int c = 0; c < 8; ++c) { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc[0][c]), _mm256_extractf128_ps(acc[0][c], 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc[1][c]), _mm256_extractf128_ps(acc[1][c], 1)); + s_row0[c] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[c] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + } + } + } + + return; + } +#endif // defined( __AVX2__ ) || defined( __AVX512F__ ) + + ggml_gemm_q1_0_8x32_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK_K; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index f18758f16bb6..13ba4434e6ee 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -208,6 +208,42 @@ void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GG } } +void ggml_quantize_mat_q8_0_4x32_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + + // scalar + float srcv[4][QK8_0]; + float id[4]; + + for (int i = 0; i < nb; i++) { + for (int row_iter = 0; row_iter < 4; row_iter++) { + float amax = 0.0f; // absolute max + + for (int j = 0; j < QK8_0; j++) { + srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j]; + amax = MAX(amax, fabsf(srcv[row_iter][j])); + } + + const float d = amax / ((1 << 7) - 1); + id[row_iter] = d ? 1.0f / d : 0.0f; + + y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d); + } + + // Store each row's 32 bytes contiguously + for (int r = 0; r < 4; r++) { + for (int j = 0; j < QK8_0; j++) { + float x0 = srcv[r][j] * id[r]; + y[i].qs[r * QK8_0 + j] = roundf(x0); + } + } + } +} + void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK_K == 256); assert(k % QK_K == 0); @@ -339,6 +375,12 @@ template <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTR ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row); } +template <> void ggml_quantize_mat_t<32, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { + assert(nrow == 4); + UNUSED(nrow); + ggml_quantize_mat_q8_0_4x32(x, vy, n_per_row); +} + #if defined __riscv_zvfh template <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); @@ -884,6 +926,64 @@ void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, } } +void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + assert (n % QK1_0 == 0); + assert (nc % 8 == 0); + + UNUSED(bs); + UNUSED(nr); + + const int nb = n / QK1_0; + const int nb32 = n / QK8_0; + const int ncols8 = nc / 8; + + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + const block_q8_0 * a_ptr = (const block_q8_0 *)vy; + + for (int y = 0; y < nr; ++y) { + const block_q8_0 * a_row = a_ptr + (size_t)y * nb32; + float * row_out = s + (size_t)y * nc; + + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + + float acc[8] = {0}; + + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + + float block_acc[8] = {0}; + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * yb = &a_row[l * 4 + sb]; + const float dy = GGML_CPU_FP16_TO_FP32(yb->d); + + const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; + const int8_t * y = yb->qs; + + for (int c = 0; c < 8; ++c) { + int sumi = 0; + for (int i = 0; i < QK8_0; ++i) { + sumi += ((qs[i] >> c) & 1) ? y[i] : -y[i]; + } + block_acc[c] += dy * (float)sumi; + } + } + + for (int c = 0; c < 8; ++c) { + acc[c] += bd[c] * block_acc[c]; + } + } + + for (int c = 0; c < 8; ++c) { + row_out[x * 8 + c] = acc[c]; + } + } + } +} + void ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK_K; const int nb = n / qk; @@ -1819,6 +1919,70 @@ void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, } } +void ggml_gemm_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + assert (n % QK1_0 == 0); + assert (nr % 4 == 0); + assert (nc % 8 == 0); + + const int nb = n / QK1_0; + const int nb_q8_0 = n / QK8_0; + const int ncols8 = nc / 8; + const int nrows4 = nr / 4; + + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + + for (int y = 0; y < nrows4; ++y) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *)vy + (y * nb_q8_0); + + for (int row_base = 0; row_base < 4; row_base += 2) { + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + + float acc[2][8] = {{0}}; + + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + + float block_acc[2][8] = {{0}}; + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; + const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; + + for (int r = 0; r < 2; ++r) { + const float dy = GGML_CPU_FP16_TO_FP32(yb->d[row_base + r]); + + for (int c = 0; c < 8; ++c) { + int sumi = 0; + for (int i = 0; i < QK8_0; ++i) { + const int8_t y_val = yb->qs[(row_base + r) * 32 + i]; + sumi += ((qs[i] >> c) & 1) ? y_val : -y_val; + } + block_acc[r][c] += dy * (float)sumi; + } + } + } + + for (int r = 0; r < 2; ++r) { + for (int c = 0; c < 8; ++c) { + acc[r][c] += bd[c] * block_acc[r][c]; + } + } + } + + for (int r = 0; r < 2; ++r) { + float * row_out = s + (y * 4 + row_base + r) * bs + x * 8; + for (int c = 0; c < 8; ++c) { + row_out[c] = acc[r][c]; + } + } + } + } + } +} + void ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK_K; const int nb = n / qk; @@ -2808,6 +2972,29 @@ static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_in return out; } +static block_q1_0x8 make_block_q1_0x8(block_q1_0 * in, unsigned int blck_size_interleave) { + block_q1_0x8 out; + + GGML_ASSERT(blck_size_interleave == 8); + + for (int i = 0; i < 8; ++i) { + out.d[i] = in[i].d; + } + + for (int sb = 0; sb < 4; ++sb) { + for (int i = 0; i < 32; ++i) { + uint8_t byte = 0; + for (int c = 0; c < 8; ++c) { + uint8_t src = in[c].qs[sb * 4 + i / 8]; + byte |= ((src >> (i % 8)) & 1) << c; + } + out.qs[sb * 32 + i] = byte; + } + } + + return out; +} + static block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) { block_q4_0x16 out; @@ -3477,6 +3664,36 @@ static int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block GGML_UNUSED(data_size); } +static int repack_q1_0_to_q1_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q1_0); + GGML_ASSERT(interleave_block == 8); + constexpr int nrows_interleaved = 8; + + block_q1_0x8 * dst = (block_q1_0x8 *) t->data; + const block_q1_0 * src = (const block_q1_0 *) data; + block_q1_0 dst_tmp[8]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK1_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q1_0)); + + if (t->ne[1] % nrows_interleaved != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; ++x) { + for (int i = 0; i < nrows_interleaved; ++i) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q1_0x8(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + + return 0; +} + static int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, @@ -3934,6 +4151,10 @@ template <> int repack(struct ggml_tensor * t, const void * da return repack_q8_0_to_q8_0_4_bl(t, 8, data, data_size); } +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_8_bl(t, 8, data, data_size); +} + #if defined __riscv_zvfh template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_q4_0_to_q4_0_16_bl(t, 1, data, data_size); @@ -4031,6 +4252,10 @@ template <> void gemv(int n, float * s, size_t ggml_gemv_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_8x32_q8_0(n, s, bs, vx, vy, nr, nc); +} + #if defined __riscv_zvfh template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc); @@ -4128,6 +4353,10 @@ template <> void gemm(int n, float * s, size_t ggml_gemm_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_8x32_q8_0(n, s, bs, vx, vy, nr, nc); +} + #if defined __riscv_zvfh template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc); @@ -4558,6 +4787,9 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons static const ggml::cpu::repack::tensor_traits q8_0_4x4_q8_0; static const ggml::cpu::repack::tensor_traits q8_0_4x8_q8_0; + // instance for Q1_0 + static const ggml::cpu::repack::tensor_traits q1_0_8x32_q8_0; + // instances for RISC-V // // These implement outer-product style matrix multiplication kernels with @@ -4597,6 +4829,12 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons } #endif } + } else if (cur->type == GGML_TYPE_Q1_0) { + if (ggml_cpu_has_avx2()) { + if (cur->ne[1] % 8 == 0) { + return &q1_0_8x32_q8_0; + } + } } else if (cur->type == GGML_TYPE_Q4_K) { if (ggml_cpu_has_avx2()) { if (cur->ne[1] % 8 == 0) { diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index cb21edf62394..5db7ca8269ff 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -11,6 +11,9 @@ ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void); template constexpr int QK_0() { + if constexpr (K == 1) { + return QK1_0; + } if constexpr (K == 4) { return QK4_0; } @@ -32,7 +35,9 @@ static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, "wrong static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong block<8,4> size/padding"); static_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding"); static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding"); +static_assert(sizeof(block<1, 8>) == 8 * sizeof(ggml_half) + QK1_0, "wrong block<1,8> size/padding"); +using block_q1_0x8 = block<1, 8>; using block_q4_0x4 = block<4, 4>; using block_q4_0x8 = block<4, 8>; using block_q4_0x16 = block<4, 16>; @@ -139,6 +144,7 @@ extern "C" { void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_quantize_mat_q8_0_4x32(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -157,6 +163,7 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -173,6 +180,7 @@ void ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); @@ -191,6 +199,7 @@ void ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v // Native implementations void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_quantize_mat_q8_0_4x32_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -209,6 +218,7 @@ void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -225,6 +235,7 @@ void ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); From 5a9ab761476ae224d6b146db3cc3642d02fcd787 Mon Sep 17 00:00:00 2001 From: pl752 Date: Tue, 5 May 2026 19:38:49 +0500 Subject: [PATCH 08/11] Corrected gemv with assumption of nr==1 for consistency --- ggml/src/ggml-cpu/arch/x86/repack.cpp | 165 +++++++++++++------------- ggml/src/ggml-cpu/repack.cpp | 52 ++++---- 2 files changed, 103 insertions(+), 114 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index d97c90c21c7a..f2d5c9836309 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -1535,12 +1535,12 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v { assert (n % QK1_0 == 0); assert (nc % 8 == 0); + assert (nr == 1); UNUSED(bs); UNUSED(nr); const int nb = n / QK1_0; - const int nb32 = n / QK8_0; const int ncols8 = nc / 8; const __m256i ones_8 = _mm256_set1_epi8(1); @@ -1548,28 +1548,28 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i zero = _mm256_setzero_si256(); // Shuffle LUTs for columns 0-3: LUT[b & 0xF] = (b >> c) & 1 ? 0x00 : 0xFF - alignas(32) static const uint8_t sm_lut_c0[16] = { + alignas(16) static const uint8_t sm_lut_c0[16] = { 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00 }; - alignas(32) static const uint8_t sm_lut_c1[16] = { + alignas(16) static const uint8_t sm_lut_c1[16] = { 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00 }; - alignas(32) static const uint8_t sm_lut_c2[16] = { + alignas(16) static const uint8_t sm_lut_c2[16] = { 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00 }; - alignas(32) static const uint8_t sm_lut_c3[16] = { + alignas(16) static const uint8_t sm_lut_c3[16] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; const __m256i lut[4] = { - _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c0)), - _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c1)), - _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c2)), - _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)sm_lut_c3)), + _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c0)), + _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c1)), + _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c2)), + _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c3)), }; // Column masks for columns 4-7 (AND+cmpeq path) @@ -1581,87 +1581,82 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; const block_q8_0 * a_ptr = (const block_q8_0 *)vy; - for (int y = 0; y < nr; ++y) { - const block_q8_0 * a_row = a_ptr + (size_t)y * nb32; - float * row_out = s + (size_t)y * nc; - - for (int x = 0; x < ncols8; ++x) { - const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - - __m256 acc[8]; - for (int c = 0; c < 8; ++c) acc[c] = _mm256_setzero_ps(); - - for (int l = 0; l < nb; ++l) { - float bd[8]; - for (int c = 0; c < 8; ++c) - bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); - - __m256 block_acc[8]; - for (int c = 0; c < 8; ++c) block_acc[c] = _mm256_setzero_ps(); - - const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; - - for (int sb = 0; sb < 4; ++sb) { - const block_q8_0 * yb = &a_row[l * 4 + sb]; - const __m256i rhs = _mm256_loadu_si256((const __m256i *)yb->qs); - const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); - - const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); - - // Columns 0-3: shuffle LUT on low 7 bits - const __m256i qs_lo7 = _mm256_and_si256(qs_vec, _mm256_set1_epi8(0x7F)); - const __m256i sm0 = _mm256_shuffle_epi8(lut[0], qs_lo7); - const __m256i sm1 = _mm256_shuffle_epi8(lut[1], qs_lo7); - const __m256i sm2 = _mm256_shuffle_epi8(lut[2], qs_lo7); - const __m256i sm3 = _mm256_shuffle_epi8(lut[3], qs_lo7); - - // Columns 4-7: AND + cmpeq - const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_4), zero); - const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_5), zero); - const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_6), zero); - const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_7), zero); - - // Sign-flip and accumulate for all 8 columns - const __m256i sy0 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm0), sm0); - const __m256i sy1 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm1), sm1); - const __m256i sy2 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm2), sm2); - const __m256i sy3 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm3), sm3); - const __m256i sy4 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm4), sm4); - const __m256i sy5 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm5), sm5); - const __m256i sy6 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm6), sm6); - const __m256i sy7 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm7), sm7); - - const __m256i s32_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16); - const __m256i s32_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16); - const __m256i s32_2 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16); - const __m256i s32_3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16); - const __m256i s32_4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16); - const __m256i s32_5 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16); - const __m256i s32_6 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16); - const __m256i s32_7 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16); - - block_acc[0] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_0), block_acc[0]); - block_acc[1] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_1), block_acc[1]); - block_acc[2] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_2), block_acc[2]); - block_acc[3] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_3), block_acc[3]); - block_acc[4] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_4), block_acc[4]); - block_acc[5] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_5), block_acc[5]); - block_acc[6] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_6), block_acc[6]); - block_acc[7] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_7), block_acc[7]); - } - - for (int c = 0; c < 8; ++c) { - acc[c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[c], acc[c]); - } + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + + __m256 acc[8]; + for (int c = 0; c < 8; ++c) acc[c] = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + + __m256 block_acc[8]; + for (int c = 0; c < 8; ++c) block_acc[c] = _mm256_setzero_ps(); + + const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * yb = &a_ptr[l * 4 + sb]; + const __m256i rhs = _mm256_loadu_si256((const __m256i *)yb->qs); + const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + + const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); + + // Columns 0-3: shuffle LUT on low 7 bits + const __m256i qs_lo7 = _mm256_and_si256(qs_vec, _mm256_set1_epi8(0x7F)); + const __m256i sm0 = _mm256_shuffle_epi8(lut[0], qs_lo7); + const __m256i sm1 = _mm256_shuffle_epi8(lut[1], qs_lo7); + const __m256i sm2 = _mm256_shuffle_epi8(lut[2], qs_lo7); + const __m256i sm3 = _mm256_shuffle_epi8(lut[3], qs_lo7); + + // Columns 4-7: AND + cmpeq + const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_4), zero); + const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_5), zero); + const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_6), zero); + const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_7), zero); + + // Sign-flip and accumulate for all 8 columns + const __m256i sy0 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm0), sm0); + const __m256i sy1 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm1), sm1); + const __m256i sy2 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm2), sm2); + const __m256i sy3 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm3), sm3); + const __m256i sy4 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm4), sm4); + const __m256i sy5 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm5), sm5); + const __m256i sy6 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm6), sm6); + const __m256i sy7 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm7), sm7); + + const __m256i s32_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16); + const __m256i s32_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16); + const __m256i s32_2 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16); + const __m256i s32_3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16); + const __m256i s32_4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16); + const __m256i s32_5 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16); + const __m256i s32_6 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16); + const __m256i s32_7 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16); + + block_acc[0] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_0), block_acc[0]); + block_acc[1] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_1), block_acc[1]); + block_acc[2] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_2), block_acc[2]); + block_acc[3] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_3), block_acc[3]); + block_acc[4] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_4), block_acc[4]); + block_acc[5] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_5), block_acc[5]); + block_acc[6] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_6), block_acc[6]); + block_acc[7] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_7), block_acc[7]); } - // Reduce 8 lanes to 1 value per column and store for (int c = 0; c < 8; ++c) { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc[c]), _mm256_extractf128_ps(acc[c], 1)); - const __m128 t = _mm_hadd_ps(v, v); - row_out[x * 8 + c] = _mm_cvtss_f32(_mm_hadd_ps(t, t)); + acc[c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[c], acc[c]); } } + + // Reduce 8 lanes to 1 value per column and store + for (int c = 0; c < 8; ++c) { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc[c]), _mm256_extractf128_ps(acc[c], 1)); + const __m128 t = _mm_hadd_ps(v, v); + s[x * 8 + c] = _mm_cvtss_f32(_mm_hadd_ps(t, t)); + } } return; diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 13ba4434e6ee..7b0a0a1a8a09 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -929,58 +929,52 @@ void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { assert (n % QK1_0 == 0); assert (nc % 8 == 0); + assert (nr == 1); UNUSED(bs); UNUSED(nr); const int nb = n / QK1_0; - const int nb32 = n / QK8_0; const int ncols8 = nc / 8; const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; const block_q8_0 * a_ptr = (const block_q8_0 *)vy; - for (int y = 0; y < nr; ++y) { - const block_q8_0 * a_row = a_ptr + (size_t)y * nb32; - float * row_out = s + (size_t)y * nc; + for (int x = 0; x < ncols8; ++x) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - for (int x = 0; x < ncols8; ++x) { - const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; + float acc[8] = {0}; - float acc[8] = {0}; + for (int l = 0; l < nb; ++l) { + float bd[8]; + for (int c = 0; c < 8; ++c) + bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); - for (int l = 0; l < nb; ++l) { - float bd[8]; - for (int c = 0; c < 8; ++c) - bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + float block_acc[8] = {0}; - float block_acc[8] = {0}; + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * yb = &a_ptr[l * 4 + sb]; + const float dy = GGML_CPU_FP16_TO_FP32(yb->d); - for (int sb = 0; sb < 4; ++sb) { - const block_q8_0 * yb = &a_row[l * 4 + sb]; - const float dy = GGML_CPU_FP16_TO_FP32(yb->d); - - const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; - const int8_t * y = yb->qs; - - for (int c = 0; c < 8; ++c) { - int sumi = 0; - for (int i = 0; i < QK8_0; ++i) { - sumi += ((qs[i] >> c) & 1) ? y[i] : -y[i]; - } - block_acc[c] += dy * (float)sumi; - } - } + const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; + const int8_t * y = yb->qs; for (int c = 0; c < 8; ++c) { - acc[c] += bd[c] * block_acc[c]; + int sumi = 0; + for (int i = 0; i < QK8_0; ++i) { + sumi += ((qs[i] >> c) & 1) ? y[i] : -y[i]; + } + block_acc[c] += dy * (float)sumi; } } for (int c = 0; c < 8; ++c) { - row_out[x * 8 + c] = acc[c]; + acc[c] += bd[c] * block_acc[c]; } } + + static_assert(sizeof(acc) == 32); + memcpy(s + x*8, acc, sizeof(acc)); } } From 3fa569db05ca2e65d691d5a74133f2e99451f5b5 Mon Sep 17 00:00:00 2001 From: pl752 Date: Tue, 5 May 2026 20:51:50 +0500 Subject: [PATCH 09/11] Unrolled mm256 registers to avoid register pointers --- ggml/src/ggml-cpu/arch/x86/repack.cpp | 257 ++++++++++++++++++-------- 1 file changed, 179 insertions(+), 78 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index f2d5c9836309..768a0952bf8e 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -1565,12 +1565,10 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - const __m256i lut[4] = { - _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c0)), - _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c1)), - _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c2)), - _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c3)), - }; + const __m256i lut0 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c0)); + const __m256i lut1 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c1)); + const __m256i lut2 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c2)); + const __m256i lut3 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c3)); // Column masks for columns 4-7 (AND+cmpeq path) const __m256i col_mask_4 = _mm256_set1_epi8(16); @@ -1578,22 +1576,37 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i col_mask_6 = _mm256_set1_epi8(64); const __m256i col_mask_7 = _mm256_set1_epi8((int8_t)-128); + const __m256i low_nibble_mask = _mm256_set1_epi8(0x0F); + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; const block_q8_0 * a_ptr = (const block_q8_0 *)vy; for (int x = 0; x < ncols8; ++x) { const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - __m256 acc[8]; - for (int c = 0; c < 8; ++c) acc[c] = _mm256_setzero_ps(); + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + __m256 acc4 = _mm256_setzero_ps(); + __m256 acc5 = _mm256_setzero_ps(); + __m256 acc6 = _mm256_setzero_ps(); + __m256 acc7 = _mm256_setzero_ps(); for (int l = 0; l < nb; ++l) { float bd[8]; - for (int c = 0; c < 8; ++c) + for (int c = 0; c < 8; ++c) { bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + } - __m256 block_acc[8]; - for (int c = 0; c < 8; ++c) block_acc[c] = _mm256_setzero_ps(); + __m256 ba0 = _mm256_setzero_ps(); + __m256 ba1 = _mm256_setzero_ps(); + __m256 ba2 = _mm256_setzero_ps(); + __m256 ba3 = _mm256_setzero_ps(); + __m256 ba4 = _mm256_setzero_ps(); + __m256 ba5 = _mm256_setzero_ps(); + __m256 ba6 = _mm256_setzero_ps(); + __m256 ba7 = _mm256_setzero_ps(); const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; @@ -1605,11 +1618,11 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); // Columns 0-3: shuffle LUT on low 7 bits - const __m256i qs_lo7 = _mm256_and_si256(qs_vec, _mm256_set1_epi8(0x7F)); - const __m256i sm0 = _mm256_shuffle_epi8(lut[0], qs_lo7); - const __m256i sm1 = _mm256_shuffle_epi8(lut[1], qs_lo7); - const __m256i sm2 = _mm256_shuffle_epi8(lut[2], qs_lo7); - const __m256i sm3 = _mm256_shuffle_epi8(lut[3], qs_lo7); + const __m256i qs_lo4 = _mm256_and_si256(qs_vec, low_nibble_mask); + const __m256i sm0 = _mm256_shuffle_epi8(lut0, qs_lo4); + const __m256i sm1 = _mm256_shuffle_epi8(lut1, qs_lo4); + const __m256i sm2 = _mm256_shuffle_epi8(lut2, qs_lo4); + const __m256i sm3 = _mm256_shuffle_epi8(lut3, qs_lo4); // Columns 4-7: AND + cmpeq const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_4), zero); @@ -1627,35 +1640,57 @@ void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i sy6 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm6), sm6); const __m256i sy7 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm7), sm7); - const __m256i s32_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16); - const __m256i s32_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16); - const __m256i s32_2 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16); - const __m256i s32_3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16); - const __m256i s32_4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16); - const __m256i s32_5 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16); - const __m256i s32_6 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16); - const __m256i s32_7 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16); - - block_acc[0] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_0), block_acc[0]); - block_acc[1] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_1), block_acc[1]); - block_acc[2] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_2), block_acc[2]); - block_acc[3] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_3), block_acc[3]); - block_acc[4] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_4), block_acc[4]); - block_acc[5] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_5), block_acc[5]); - block_acc[6] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_6), block_acc[6]); - block_acc[7] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32_7), block_acc[7]); + ba0 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16)), ba0); + ba1 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16)), ba1); + ba2 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16)), ba2); + ba3 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16)), ba3); + ba4 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16)), ba4); + ba5 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16)), ba5); + ba6 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16)), ba6); + ba7 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16)), ba7); } - for (int c = 0; c < 8; ++c) { - acc[c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[c], acc[c]); - } + acc0 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba0, acc0); + acc1 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba1, acc1); + acc2 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba2, acc2); + acc3 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba3, acc3); + acc4 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba4, acc4); + acc5 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba5, acc5); + acc6 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba6, acc6); + acc7 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba7, acc7); } - // Reduce 8 lanes to 1 value per column and store - for (int c = 0; c < 8; ++c) { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc[c]), _mm256_extractf128_ps(acc[c], 1)); - const __m128 t = _mm_hadd_ps(v, v); - s[x * 8 + c] = _mm_cvtss_f32(_mm_hadd_ps(t, t)); + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + s[x * 8 + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc1), _mm256_extractf128_ps(acc1, 1)); + s[x * 8 + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc2), _mm256_extractf128_ps(acc2, 1)); + s[x * 8 + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc3), _mm256_extractf128_ps(acc3, 1)); + s[x * 8 + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc4), _mm256_extractf128_ps(acc4, 1)); + s[x * 8 + 4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc5), _mm256_extractf128_ps(acc5, 1)); + s[x * 8 + 5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc6), _mm256_extractf128_ps(acc6, 1)); + s[x * 8 + 6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); + } + { + const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc7), _mm256_extractf128_ps(acc7, 1)); + s[x * 8 + 7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); } } @@ -2262,8 +2297,6 @@ void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i ones_16 = _mm256_set1_epi16(1); const __m256i zero = _mm256_setzero_si256(); - const uint8_t col_masks[8] = {1, 2, 4, 8, 16, 32, 64, 128}; - const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; for (int y = 0; y < nrows4; ++y) { @@ -2273,20 +2306,21 @@ void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v for (int x = 0; x < ncols8; ++x) { const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - __m256 acc[2][8]; - for (int r = 0; r < 2; ++r) - for (int c = 0; c < 8; ++c) - acc[r][c] = _mm256_setzero_ps(); + __m256 acc00 = _mm256_setzero_ps(), acc01 = _mm256_setzero_ps(), acc02 = _mm256_setzero_ps(), acc03 = _mm256_setzero_ps(); + __m256 acc04 = _mm256_setzero_ps(), acc05 = _mm256_setzero_ps(), acc06 = _mm256_setzero_ps(), acc07 = _mm256_setzero_ps(); + __m256 acc10 = _mm256_setzero_ps(), acc11 = _mm256_setzero_ps(), acc12 = _mm256_setzero_ps(), acc13 = _mm256_setzero_ps(); + __m256 acc14 = _mm256_setzero_ps(), acc15 = _mm256_setzero_ps(), acc16 = _mm256_setzero_ps(), acc17 = _mm256_setzero_ps(); for (int l = 0; l < nb; ++l) { float bd[8]; - for (int c = 0; c < 8; ++c) + for (int c = 0; c < 8; ++c) { bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); + } - __m256 block_acc[2][8]; - for (int r = 0; r < 2; ++r) - for (int c = 0; c < 8; ++c) - block_acc[r][c] = _mm256_setzero_ps(); + __m256 ba00 = _mm256_setzero_ps(), ba01 = _mm256_setzero_ps(), ba02 = _mm256_setzero_ps(), ba03 = _mm256_setzero_ps(); + __m256 ba04 = _mm256_setzero_ps(), ba05 = _mm256_setzero_ps(), ba06 = _mm256_setzero_ps(), ba07 = _mm256_setzero_ps(); + __m256 ba10 = _mm256_setzero_ps(), ba11 = _mm256_setzero_ps(), ba12 = _mm256_setzero_ps(), ba13 = _mm256_setzero_ps(); + __m256 ba14 = _mm256_setzero_ps(), ba15 = _mm256_setzero_ps(), ba16 = _mm256_setzero_ps(), ba17 = _mm256_setzero_ps(); const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; @@ -2294,38 +2328,105 @@ void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); - __m256i sm[8]; - for (int c = 0; c < 8; ++c) { - const __m256i mask_c = _mm256_set1_epi8((int8_t)col_masks[c]); - sm[c] = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, mask_c), zero); - } - - for (int r = 0; r < 2; ++r) { - const __m256i rhs = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + r) * 32)); - const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + r])); - - for (int c = 0; c < 8; ++c) { - const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm[c]), sm[c]); - const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); - block_acc[r][c] = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(s32), block_acc[r][c]); - } - } + const __m256i sm0 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(1)), zero); + const __m256i sm1 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(2)), zero); + const __m256i sm2 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(4)), zero); + const __m256i sm3 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(8)), zero); + const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(16)), zero); + const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(32)), zero); + const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(64)), zero); + const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8((int8_t)-128)), zero); + + const __m256i rhs0 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 0) * 32)); + const __m256 dy0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 0])); + ba00 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm0), sm0)), ones_16)), ba00); + ba01 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm1), sm1)), ones_16)), ba01); + ba02 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm2), sm2)), ones_16)), ba02); + ba03 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm3), sm3)), ones_16)), ba03); + ba04 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm4), sm4)), ones_16)), ba04); + ba05 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm5), sm5)), ones_16)), ba05); + ba06 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm6), sm6)), ones_16)), ba06); + ba07 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm7), sm7)), ones_16)), ba07); + + const __m256i rhs1 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 1) * 32)); + const __m256 dy1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 1])); + ba10 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm0), sm0)), ones_16)), ba10); + ba11 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm1), sm1)), ones_16)), ba11); + ba12 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm2), sm2)), ones_16)), ba12); + ba13 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm3), sm3)), ones_16)), ba13); + ba14 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm4), sm4)), ones_16)), ba14); + ba15 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm5), sm5)), ones_16)), ba15); + ba16 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm6), sm6)), ones_16)), ba16); + ba17 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm7), sm7)), ones_16)), ba17); } - for (int r = 0; r < 2; ++r) { - for (int c = 0; c < 8; ++c) { - acc[r][c] = _mm256_fmadd_ps(_mm256_set1_ps(bd[c]), block_acc[r][c], acc[r][c]); - } - } + acc00 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba00, acc00); + acc01 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba01, acc01); + acc02 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba02, acc02); + acc03 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba03, acc03); + acc04 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba04, acc04); + acc05 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba05, acc05); + acc06 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba06, acc06); + acc07 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba07, acc07); + acc10 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba10, acc10); + acc11 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba11, acc11); + acc12 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba12, acc12); + acc13 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba13, acc13); + acc14 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba14, acc14); + acc15 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba15, acc15); + acc16 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba16, acc16); + acc17 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba17, acc17); } float * s_row0 = s + (y * 4 + row_base + 0) * bs + x * 8; float * s_row1 = s + (y * 4 + row_base + 1) * bs + x * 8; - for (int c = 0; c < 8; ++c) { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc[0][c]), _mm256_extractf128_ps(acc[0][c], 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc[1][c]), _mm256_extractf128_ps(acc[1][c], 1)); - s_row0[c] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[c] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc00), _mm256_extractf128_ps(acc00, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc10), _mm256_extractf128_ps(acc10, 1)); + s_row0[0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc01), _mm256_extractf128_ps(acc01, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc11), _mm256_extractf128_ps(acc11, 1)); + s_row0[1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc02), _mm256_extractf128_ps(acc02, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc12), _mm256_extractf128_ps(acc12, 1)); + s_row0[2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc03), _mm256_extractf128_ps(acc03, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc13), _mm256_extractf128_ps(acc13, 1)); + s_row0[3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc04), _mm256_extractf128_ps(acc04, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc14), _mm256_extractf128_ps(acc14, 1)); + s_row0[4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc05), _mm256_extractf128_ps(acc05, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc15), _mm256_extractf128_ps(acc15, 1)); + s_row0[5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc06), _mm256_extractf128_ps(acc06, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc16), _mm256_extractf128_ps(acc16, 1)); + s_row0[6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); + } + { + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc07), _mm256_extractf128_ps(acc07, 1)); + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc17), _mm256_extractf128_ps(acc17, 1)); + s_row0[7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); + s_row1[7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); } } } From 4e34da1643c12e67bd99bb90012f41526e5bfbec Mon Sep 17 00:00:00 2001 From: pl752 Date: Tue, 5 May 2026 23:01:28 +0500 Subject: [PATCH 10/11] Split 8 cols to two passes to reduce register pressure --- ggml/src/ggml-cpu/arch/x86/repack.cpp | 201 ++++++++++---------------- 1 file changed, 77 insertions(+), 124 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index 768a0952bf8e..c783f0cc063f 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -2279,6 +2279,77 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } +#define GGML_GEMM_Q1_0_4COL(M0, M1, M2, M3, D0, D1, D2, D3, OFF) \ + { \ + const __m256i cm0 = _mm256_set1_epi8(M0); \ + const __m256i cm1 = _mm256_set1_epi8(M1); \ + const __m256i cm2 = _mm256_set1_epi8(M2); \ + const __m256i cm3 = _mm256_set1_epi8(M3); \ + __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(), a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps(); \ + __m256 a10 = _mm256_setzero_ps(), a11 = _mm256_setzero_ps(), a12 = _mm256_setzero_ps(), a13 = _mm256_setzero_ps(); \ + for (int l = 0; l < nb; ++l) { \ + const float bd0 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D0]); \ + const float bd1 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D1]); \ + const float bd2 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D2]); \ + const float bd3 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D3]); \ + __m256 b0 = _mm256_setzero_ps(), b1 = _mm256_setzero_ps(), b2 = _mm256_setzero_ps(), b3 = _mm256_setzero_ps(); \ + __m256 b10 = _mm256_setzero_ps(), b11 = _mm256_setzero_ps(), b12 = _mm256_setzero_ps(), b13 = _mm256_setzero_ps(); \ + const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; \ + for (int sb = 0; sb < 4; ++sb) { \ + const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; \ + const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); \ + const __m256i sm0 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm0), zero); \ + const __m256i sm1 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm1), zero); \ + const __m256i sm2 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm2), zero); \ + const __m256i sm3 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm3), zero); \ + const __m256i rhs0 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 0) * 32)); \ + const __m256 dy0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 0])); \ + b0 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm0), sm0)), ones_16)), b0); \ + b1 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm1), sm1)), ones_16)), b1); \ + b2 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm2), sm2)), ones_16)), b2); \ + b3 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm3), sm3)), ones_16)), b3); \ + const __m256i rhs1 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 1) * 32)); \ + const __m256 dy1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 1])); \ + b10 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm0), sm0)), ones_16)), b10); \ + b11 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm1), sm1)), ones_16)), b11); \ + b12 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm2), sm2)), ones_16)), b12); \ + b13 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm3), sm3)), ones_16)), b13); \ + } \ + a0 = _mm256_fmadd_ps(_mm256_set1_ps(bd0), b0, a0); \ + a1 = _mm256_fmadd_ps(_mm256_set1_ps(bd1), b1, a1); \ + a2 = _mm256_fmadd_ps(_mm256_set1_ps(bd2), b2, a2); \ + a3 = _mm256_fmadd_ps(_mm256_set1_ps(bd3), b3, a3); \ + a10 = _mm256_fmadd_ps(_mm256_set1_ps(bd0), b10, a10); \ + a11 = _mm256_fmadd_ps(_mm256_set1_ps(bd1), b11, a11); \ + a12 = _mm256_fmadd_ps(_mm256_set1_ps(bd2), b12, a12); \ + a13 = _mm256_fmadd_ps(_mm256_set1_ps(bd3), b13, a13); \ + } \ + { \ + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); \ + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a10), _mm256_extractf128_ps(a10, 1)); \ + s_row0[OFF + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ + s_row1[OFF + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ + } \ + { \ + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a1), _mm256_extractf128_ps(a1, 1)); \ + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a11), _mm256_extractf128_ps(a11, 1)); \ + s_row0[OFF + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ + s_row1[OFF + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ + } \ + { \ + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); \ + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a12), _mm256_extractf128_ps(a12, 1)); \ + s_row0[OFF + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ + s_row1[OFF + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ + } \ + { \ + const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a3), _mm256_extractf128_ps(a3, 1)); \ + const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a13), _mm256_extractf128_ps(a13, 1)); \ + s_row0[OFF + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ + s_row1[OFF + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ + } \ + } + void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { #if defined( __AVX2__ ) || defined( __AVX512F__ ) { @@ -2303,131 +2374,13 @@ void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const block_q8_0x4 * a_ptr = (const block_q8_0x4 *)vy + (y * nb_q8_0); for (int row_base = 0; row_base < 4; row_base += 2) { - for (int x = 0; x < ncols8; ++x) { - const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - - __m256 acc00 = _mm256_setzero_ps(), acc01 = _mm256_setzero_ps(), acc02 = _mm256_setzero_ps(), acc03 = _mm256_setzero_ps(); - __m256 acc04 = _mm256_setzero_ps(), acc05 = _mm256_setzero_ps(), acc06 = _mm256_setzero_ps(), acc07 = _mm256_setzero_ps(); - __m256 acc10 = _mm256_setzero_ps(), acc11 = _mm256_setzero_ps(), acc12 = _mm256_setzero_ps(), acc13 = _mm256_setzero_ps(); - __m256 acc14 = _mm256_setzero_ps(), acc15 = _mm256_setzero_ps(), acc16 = _mm256_setzero_ps(), acc17 = _mm256_setzero_ps(); - - for (int l = 0; l < nb; ++l) { - float bd[8]; - for (int c = 0; c < 8; ++c) { - bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); - } - - __m256 ba00 = _mm256_setzero_ps(), ba01 = _mm256_setzero_ps(), ba02 = _mm256_setzero_ps(), ba03 = _mm256_setzero_ps(); - __m256 ba04 = _mm256_setzero_ps(), ba05 = _mm256_setzero_ps(), ba06 = _mm256_setzero_ps(), ba07 = _mm256_setzero_ps(); - __m256 ba10 = _mm256_setzero_ps(), ba11 = _mm256_setzero_ps(), ba12 = _mm256_setzero_ps(), ba13 = _mm256_setzero_ps(); - __m256 ba14 = _mm256_setzero_ps(), ba15 = _mm256_setzero_ps(), ba16 = _mm256_setzero_ps(), ba17 = _mm256_setzero_ps(); - - const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; - - for (int sb = 0; sb < 4; ++sb) { - const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; - const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); - - const __m256i sm0 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(1)), zero); - const __m256i sm1 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(2)), zero); - const __m256i sm2 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(4)), zero); - const __m256i sm3 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(8)), zero); - const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(16)), zero); - const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(32)), zero); - const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8(64)), zero); - const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, _mm256_set1_epi8((int8_t)-128)), zero); - - const __m256i rhs0 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 0) * 32)); - const __m256 dy0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 0])); - ba00 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm0), sm0)), ones_16)), ba00); - ba01 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm1), sm1)), ones_16)), ba01); - ba02 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm2), sm2)), ones_16)), ba02); - ba03 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm3), sm3)), ones_16)), ba03); - ba04 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm4), sm4)), ones_16)), ba04); - ba05 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm5), sm5)), ones_16)), ba05); - ba06 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm6), sm6)), ones_16)), ba06); - ba07 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm7), sm7)), ones_16)), ba07); - - const __m256i rhs1 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 1) * 32)); - const __m256 dy1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 1])); - ba10 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm0), sm0)), ones_16)), ba10); - ba11 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm1), sm1)), ones_16)), ba11); - ba12 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm2), sm2)), ones_16)), ba12); - ba13 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm3), sm3)), ones_16)), ba13); - ba14 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm4), sm4)), ones_16)), ba14); - ba15 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm5), sm5)), ones_16)), ba15); - ba16 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm6), sm6)), ones_16)), ba16); - ba17 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm7), sm7)), ones_16)), ba17); - } - - acc00 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba00, acc00); - acc01 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba01, acc01); - acc02 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba02, acc02); - acc03 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba03, acc03); - acc04 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba04, acc04); - acc05 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba05, acc05); - acc06 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba06, acc06); - acc07 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba07, acc07); - acc10 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba10, acc10); - acc11 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba11, acc11); - acc12 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba12, acc12); - acc13 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba13, acc13); - acc14 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba14, acc14); - acc15 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba15, acc15); - acc16 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba16, acc16); - acc17 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba17, acc17); - } + for (int x8 = 0; x8 < ncols8; ++x8) { + const block_q1_0x8 * b_ptr = vx_bi + (size_t)x8 * nb; + float * s_row0 = s + (y * 4 + row_base + 0) * bs + x8 * 8; + float * s_row1 = s + (y * 4 + row_base + 1) * bs + x8 * 8; - float * s_row0 = s + (y * 4 + row_base + 0) * bs + x * 8; - float * s_row1 = s + (y * 4 + row_base + 1) * bs + x * 8; - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc00), _mm256_extractf128_ps(acc00, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc10), _mm256_extractf128_ps(acc10, 1)); - s_row0[0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc01), _mm256_extractf128_ps(acc01, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc11), _mm256_extractf128_ps(acc11, 1)); - s_row0[1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc02), _mm256_extractf128_ps(acc02, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc12), _mm256_extractf128_ps(acc12, 1)); - s_row0[2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc03), _mm256_extractf128_ps(acc03, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc13), _mm256_extractf128_ps(acc13, 1)); - s_row0[3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc04), _mm256_extractf128_ps(acc04, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc14), _mm256_extractf128_ps(acc14, 1)); - s_row0[4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc05), _mm256_extractf128_ps(acc05, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc15), _mm256_extractf128_ps(acc15, 1)); - s_row0[5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc06), _mm256_extractf128_ps(acc06, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc16), _mm256_extractf128_ps(acc16, 1)); - s_row0[6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } - { - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(acc07), _mm256_extractf128_ps(acc07, 1)); - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(acc17), _mm256_extractf128_ps(acc17, 1)); - s_row0[7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); - s_row1[7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); - } + GGML_GEMM_Q1_0_4COL(1, 2, 4, 8, 0, 1, 2, 3, 0) + GGML_GEMM_Q1_0_4COL(16, 32, 64, -128, 4, 5, 6, 7, 4) } } } From a508eee41c8dc739bcc0a74f9b0f071bed29143c Mon Sep 17 00:00:00 2001 From: pl752 Date: Fri, 8 May 2026 12:51:28 +0500 Subject: [PATCH 11/11] Implement rot4 Q1.0 repack kernels with positive-sum GEMV optimization --- ggml/src/ggml-cpu/arch-fallback.h | 36 +- ggml/src/ggml-cpu/arch/x86/repack.cpp | 1002 ++++++++++++++++--------- ggml/src/ggml-cpu/repack.cpp | 152 ++-- ggml/src/ggml-cpu/repack.h | 12 +- 4 files changed, 756 insertions(+), 446 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 1e3a4521c5b8..3163e1949823 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -39,7 +39,6 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -56,7 +55,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -73,26 +72,24 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp -#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 @@ -131,7 +128,6 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -148,7 +144,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -165,7 +161,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__loongarch64) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -181,7 +177,6 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -198,7 +193,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -215,14 +210,13 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1 @@ -243,7 +237,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -259,7 +253,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__s390x__) // quants.c @@ -282,7 +276,6 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -299,7 +292,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -316,7 +309,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #elif defined(__wasm__) // quants.c #define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1 @@ -340,7 +333,6 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_quantize_mat_q8_0_4x32_generic ggml_quantize_mat_q8_0_4x32 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -357,7 +349,7 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemv_q1_0_8x32_q8_0_generic ggml_gemv_q1_0_8x32_q8_0 +#define ggml_gemv_q1_0_8x4_q8_0_generic ggml_gemv_q1_0_8x4_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -374,5 +366,5 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#define ggml_gemm_q1_0_8x32_q8_0_generic ggml_gemm_q1_0_8x32_q8_0 +#define ggml_gemm_q1_0_8x4_q8_0_generic ggml_gemm_q1_0_8x4_q8_0 #endif diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index c783f0cc063f..4a5834ad548c 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -175,6 +175,74 @@ static inline __m256i mul_sum_i8_pairs_acc_int32x8(const __m256i acc, const __m2 } #endif +void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + +#if defined(__AVX2__) + const __m256 signBit = _mm256_set1_ps(-0.0f); + + for (int i = 0; i < nb; ++i) { + __m256 srcv[4][4]; + __m256 idv[4]; + + for (int r = 0; r < 4; ++r) { + const float *xr = x + r * k + i * 32; + + srcv[r][0] = _mm256_loadu_ps(xr + 0); + srcv[r][1] = _mm256_loadu_ps(xr + 8); + srcv[r][2] = _mm256_loadu_ps(xr + 16); + srcv[r][3] = _mm256_loadu_ps(xr + 24); + + __m256 maxAbs = _mm256_andnot_ps(signBit, srcv[r][0]); + maxAbs = _mm256_max_ps(maxAbs, _mm256_andnot_ps(signBit, srcv[r][1])); + maxAbs = _mm256_max_ps(maxAbs, _mm256_andnot_ps(signBit, srcv[r][2])); + maxAbs = _mm256_max_ps(maxAbs, _mm256_andnot_ps(signBit, srcv[r][3])); + + __m128 max4 = _mm_max_ps(_mm256_castps256_ps128(maxAbs), _mm256_extractf128_ps(maxAbs, 1)); + max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); + max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4)); + + const float amax = _mm_cvtss_f32(max4); + const float d = amax / 127.0f; + const float id = amax != 0.0f ? 127.0f / amax : 0.0f; + + y[i].d[r] = GGML_CPU_FP32_TO_FP16(d); + idv[r] = _mm256_set1_ps(id); + } + + for (int j = 0; j < 4; ++j) { + __m256 q0 = _mm256_mul_ps(srcv[0][j], idv[0]); + __m256 q1 = _mm256_mul_ps(srcv[1][j], idv[1]); + __m256 q2 = _mm256_mul_ps(srcv[2][j], idv[2]); + __m256 q3 = _mm256_mul_ps(srcv[3][j], idv[3]); + + q0 = _mm256_round_ps(q0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + q1 = _mm256_round_ps(q1, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + q2 = _mm256_round_ps(q2, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + q3 = _mm256_round_ps(q3, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + __m256i i0 = _mm256_cvtps_epi32(q0); + __m256i i1 = _mm256_cvtps_epi32(q1); + __m256i i2 = _mm256_cvtps_epi32(q2); + __m256i i3 = _mm256_cvtps_epi32(q3); + + const __m256i p01 = _mm256_packs_epi32(i0, i1); + const __m256i p23 = _mm256_packs_epi32(i2, i3); + + const __m256i packed = _mm256_packs_epi16(p01, p23); + + _mm256_storeu_si256((__m256i *)(y[i].qs + j * 32), packed); + } + } +#else + ggml_quantize_mat_q8_0_4x4_generic(x, vy, k); +#endif +} + void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK8_0 == 32); assert(k % QK8_0 == 0); @@ -287,75 +355,6 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR #endif } -void ggml_quantize_mat_q8_0_4x32(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { - assert(QK8_0 == 32); - assert(k % QK8_0 == 0); - const int nb = k / QK8_0; - - block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; - -#if defined(__AVX2__) - for (int i = 0; i < nb; i++) { - for (int r = 0; r < 4; r++) { - // Load elements into 4 AVX vectors - __m256 v0 = _mm256_loadu_ps( x + r * k + i * 32 ); - __m256 v1 = _mm256_loadu_ps( x + r * k + i * 32 + 8 ); - __m256 v2 = _mm256_loadu_ps( x + r * k + i * 32 + 16 ); - __m256 v3 = _mm256_loadu_ps( x + r * k + i * 32 + 24 ); - - // Compute max(abs(e)) for the block - const __m256 signBit = _mm256_set1_ps( -0.0f ); - __m256 maxAbs = _mm256_andnot_ps( signBit, v0 ); - maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) ); - maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) ); - maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) ); - - __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); - max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); - max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); - const float maxScalar = _mm_cvtss_f32( max4 ); - - const float d = maxScalar / 127.f; - y[i].d[r] = GGML_CPU_FP32_TO_FP16(d); - const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; - const __m256 mul = _mm256_set1_ps( id ); - - // Apply multiplier - v0 = _mm256_mul_ps( v0, mul ); - v1 = _mm256_mul_ps( v1, mul ); - v2 = _mm256_mul_ps( v2, mul ); - v3 = _mm256_mul_ps( v3, mul ); - - // Round - v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); - v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); - v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); - v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); - - // Convert to int - __m256i i0 = _mm256_cvtps_epi32( v0 ); - __m256i i1 = _mm256_cvtps_epi32( v1 ); - __m256i i2 = _mm256_cvtps_epi32( v2 ); - __m256i i3 = _mm256_cvtps_epi32( v3 ); - - i0 = _mm256_packs_epi32( i0, i1 ); - i2 = _mm256_packs_epi32( i2, i3 ); - i0 = _mm256_packs_epi16( i0, i2 ); - - const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); - i0 = _mm256_permutevar8x32_epi32( i0, perm ); - - // Store row r contiguously - _mm256_storeu_si256((__m256i *)(y[i].qs + r * 32), i0); - } - } -#else - UNUSED(nb); - UNUSED(y); - ggml_quantize_mat_q8_0_4x32_generic(x, vy, k); -#endif -} - void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK_K == 256); assert(k % QK_K == 0); @@ -1530,177 +1529,6 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } -void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { -#if defined( __AVX2__ ) || defined( __AVX512F__ ) - { - assert (n % QK1_0 == 0); - assert (nc % 8 == 0); - assert (nr == 1); - - UNUSED(bs); - UNUSED(nr); - - const int nb = n / QK1_0; - const int ncols8 = nc / 8; - - const __m256i ones_8 = _mm256_set1_epi8(1); - const __m256i ones_16 = _mm256_set1_epi16(1); - const __m256i zero = _mm256_setzero_si256(); - - // Shuffle LUTs for columns 0-3: LUT[b & 0xF] = (b >> c) & 1 ? 0x00 : 0xFF - alignas(16) static const uint8_t sm_lut_c0[16] = { - 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, - 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00 - }; - alignas(16) static const uint8_t sm_lut_c1[16] = { - 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, - 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00 - }; - alignas(16) static const uint8_t sm_lut_c2[16] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00 - }; - alignas(16) static const uint8_t sm_lut_c3[16] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - }; - - const __m256i lut0 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c0)); - const __m256i lut1 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c1)); - const __m256i lut2 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c2)); - const __m256i lut3 = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i *)sm_lut_c3)); - - // Column masks for columns 4-7 (AND+cmpeq path) - const __m256i col_mask_4 = _mm256_set1_epi8(16); - const __m256i col_mask_5 = _mm256_set1_epi8(32); - const __m256i col_mask_6 = _mm256_set1_epi8(64); - const __m256i col_mask_7 = _mm256_set1_epi8((int8_t)-128); - - const __m256i low_nibble_mask = _mm256_set1_epi8(0x0F); - - const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; - const block_q8_0 * a_ptr = (const block_q8_0 *)vy; - - for (int x = 0; x < ncols8; ++x) { - const block_q1_0x8 * b_ptr = vx_bi + (size_t)x * nb; - - __m256 acc0 = _mm256_setzero_ps(); - __m256 acc1 = _mm256_setzero_ps(); - __m256 acc2 = _mm256_setzero_ps(); - __m256 acc3 = _mm256_setzero_ps(); - __m256 acc4 = _mm256_setzero_ps(); - __m256 acc5 = _mm256_setzero_ps(); - __m256 acc6 = _mm256_setzero_ps(); - __m256 acc7 = _mm256_setzero_ps(); - - for (int l = 0; l < nb; ++l) { - float bd[8]; - for (int c = 0; c < 8; ++c) { - bd[c] = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[c]); - } - - __m256 ba0 = _mm256_setzero_ps(); - __m256 ba1 = _mm256_setzero_ps(); - __m256 ba2 = _mm256_setzero_ps(); - __m256 ba3 = _mm256_setzero_ps(); - __m256 ba4 = _mm256_setzero_ps(); - __m256 ba5 = _mm256_setzero_ps(); - __m256 ba6 = _mm256_setzero_ps(); - __m256 ba7 = _mm256_setzero_ps(); - - const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; - - for (int sb = 0; sb < 4; ++sb) { - const block_q8_0 * yb = &a_ptr[l * 4 + sb]; - const __m256i rhs = _mm256_loadu_si256((const __m256i *)yb->qs); - const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); - - const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); - - // Columns 0-3: shuffle LUT on low 7 bits - const __m256i qs_lo4 = _mm256_and_si256(qs_vec, low_nibble_mask); - const __m256i sm0 = _mm256_shuffle_epi8(lut0, qs_lo4); - const __m256i sm1 = _mm256_shuffle_epi8(lut1, qs_lo4); - const __m256i sm2 = _mm256_shuffle_epi8(lut2, qs_lo4); - const __m256i sm3 = _mm256_shuffle_epi8(lut3, qs_lo4); - - // Columns 4-7: AND + cmpeq - const __m256i sm4 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_4), zero); - const __m256i sm5 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_5), zero); - const __m256i sm6 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_6), zero); - const __m256i sm7 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, col_mask_7), zero); - - // Sign-flip and accumulate for all 8 columns - const __m256i sy0 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm0), sm0); - const __m256i sy1 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm1), sm1); - const __m256i sy2 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm2), sm2); - const __m256i sy3 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm3), sm3); - const __m256i sy4 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm4), sm4); - const __m256i sy5 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm5), sm5); - const __m256i sy6 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm6), sm6); - const __m256i sy7 = _mm256_sub_epi8(_mm256_xor_si256(rhs, sm7), sm7); - - ba0 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy0), ones_16)), ba0); - ba1 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy1), ones_16)), ba1); - ba2 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy2), ones_16)), ba2); - ba3 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy3), ones_16)), ba3); - ba4 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy4), ones_16)), ba4); - ba5 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy5), ones_16)), ba5); - ba6 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy6), ones_16)), ba6); - ba7 = _mm256_fmadd_ps(dy, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy7), ones_16)), ba7); - } - - acc0 = _mm256_fmadd_ps(_mm256_set1_ps(bd[0]), ba0, acc0); - acc1 = _mm256_fmadd_ps(_mm256_set1_ps(bd[1]), ba1, acc1); - acc2 = _mm256_fmadd_ps(_mm256_set1_ps(bd[2]), ba2, acc2); - acc3 = _mm256_fmadd_ps(_mm256_set1_ps(bd[3]), ba3, acc3); - acc4 = _mm256_fmadd_ps(_mm256_set1_ps(bd[4]), ba4, acc4); - acc5 = _mm256_fmadd_ps(_mm256_set1_ps(bd[5]), ba5, acc5); - acc6 = _mm256_fmadd_ps(_mm256_set1_ps(bd[6]), ba6, acc6); - acc7 = _mm256_fmadd_ps(_mm256_set1_ps(bd[7]), ba7, acc7); - } - - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); - s[x * 8 + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc1), _mm256_extractf128_ps(acc1, 1)); - s[x * 8 + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc2), _mm256_extractf128_ps(acc2, 1)); - s[x * 8 + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc3), _mm256_extractf128_ps(acc3, 1)); - s[x * 8 + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc4), _mm256_extractf128_ps(acc4, 1)); - s[x * 8 + 4] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc5), _mm256_extractf128_ps(acc5, 1)); - s[x * 8 + 5] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc6), _mm256_extractf128_ps(acc6, 1)); - s[x * 8 + 6] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - { - const __m128 v = _mm_add_ps(_mm256_castps256_ps128(acc7), _mm256_extractf128_ps(acc7, 1)); - s[x * 8 + 7] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v, v), _mm_hadd_ps(v, v))); - } - } - - return; - } -#endif // defined( __AVX2__ ) || defined( __AVX512F__ ) - - ggml_gemv_q1_0_8x32_q8_0_generic(n, s, bs, vx, vy, nr, nc); -} - void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK_K; const int nb = n / qk; @@ -2279,117 +2107,619 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } -#define GGML_GEMM_Q1_0_4COL(M0, M1, M2, M3, D0, D1, D2, D3, OFF) \ - { \ - const __m256i cm0 = _mm256_set1_epi8(M0); \ - const __m256i cm1 = _mm256_set1_epi8(M1); \ - const __m256i cm2 = _mm256_set1_epi8(M2); \ - const __m256i cm3 = _mm256_set1_epi8(M3); \ - __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(), a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps(); \ - __m256 a10 = _mm256_setzero_ps(), a11 = _mm256_setzero_ps(), a12 = _mm256_setzero_ps(), a13 = _mm256_setzero_ps(); \ - for (int l = 0; l < nb; ++l) { \ - const float bd0 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D0]); \ - const float bd1 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D1]); \ - const float bd2 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D2]); \ - const float bd3 = GGML_CPU_FP16_TO_FP32(b_ptr[l].d[D3]); \ - __m256 b0 = _mm256_setzero_ps(), b1 = _mm256_setzero_ps(), b2 = _mm256_setzero_ps(), b3 = _mm256_setzero_ps(); \ - __m256 b10 = _mm256_setzero_ps(), b11 = _mm256_setzero_ps(), b12 = _mm256_setzero_ps(), b13 = _mm256_setzero_ps(); \ - const uint8_t * qs_base = (const uint8_t *)b_ptr[l].qs; \ - for (int sb = 0; sb < 4; ++sb) { \ - const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; \ - const __m256i qs_vec = _mm256_loadu_si256((const __m256i *)(qs_base + sb * 32)); \ - const __m256i sm0 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm0), zero); \ - const __m256i sm1 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm1), zero); \ - const __m256i sm2 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm2), zero); \ - const __m256i sm3 = _mm256_cmpeq_epi8(_mm256_and_si256(qs_vec, cm3), zero); \ - const __m256i rhs0 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 0) * 32)); \ - const __m256 dy0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 0])); \ - b0 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm0), sm0)), ones_16)), b0); \ - b1 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm1), sm1)), ones_16)), b1); \ - b2 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm2), sm2)), ones_16)), b2); \ - b3 = _mm256_fmadd_ps(dy0, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs0, sm3), sm3)), ones_16)), b3); \ - const __m256i rhs1 = _mm256_loadu_si256((const __m256i *)(yb->qs + (row_base + 1) * 32)); \ - const __m256 dy1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d[row_base + 1])); \ - b10 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm0), sm0)), ones_16)), b10); \ - b11 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm1), sm1)), ones_16)), b11); \ - b12 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm2), sm2)), ones_16)), b12); \ - b13 = _mm256_fmadd_ps(dy1, _mm256_cvtepi32_ps(_mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, _mm256_sub_epi8(_mm256_xor_si256(rhs1, sm3), sm3)), ones_16)), b13); \ - } \ - a0 = _mm256_fmadd_ps(_mm256_set1_ps(bd0), b0, a0); \ - a1 = _mm256_fmadd_ps(_mm256_set1_ps(bd1), b1, a1); \ - a2 = _mm256_fmadd_ps(_mm256_set1_ps(bd2), b2, a2); \ - a3 = _mm256_fmadd_ps(_mm256_set1_ps(bd3), b3, a3); \ - a10 = _mm256_fmadd_ps(_mm256_set1_ps(bd0), b10, a10); \ - a11 = _mm256_fmadd_ps(_mm256_set1_ps(bd1), b11, a11); \ - a12 = _mm256_fmadd_ps(_mm256_set1_ps(bd2), b12, a12); \ - a13 = _mm256_fmadd_ps(_mm256_set1_ps(bd3), b13, a13); \ - } \ - { \ - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); \ - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a10), _mm256_extractf128_ps(a10, 1)); \ - s_row0[OFF + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ - s_row1[OFF + 0] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ - } \ - { \ - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a1), _mm256_extractf128_ps(a1, 1)); \ - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a11), _mm256_extractf128_ps(a11, 1)); \ - s_row0[OFF + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ - s_row1[OFF + 1] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ - } \ - { \ - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); \ - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a12), _mm256_extractf128_ps(a12, 1)); \ - s_row0[OFF + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ - s_row1[OFF + 2] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ - } \ - { \ - const __m128 v0 = _mm_add_ps(_mm256_castps256_ps128(a3), _mm256_extractf128_ps(a3, 1)); \ - const __m128 v1 = _mm_add_ps(_mm256_castps256_ps128(a13), _mm256_extractf128_ps(a13, 1)); \ - s_row0[OFF + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v0, v0), _mm_hadd_ps(v0, v0))); \ - s_row1[OFF + 3] = _mm_cvtss_f32(_mm_hadd_ps(_mm_hadd_ps(v1, v1), _mm_hadd_ps(v1, v1))); \ - } \ +// Rotated Q1.0 layout helpers and kernels + +static const int8_t q1_byte_sel[32] = { + 0,0,0,0, 0,0,0,0, + 1,1,1,1, 1,1,1,1, + 2,2,2,2, 2,2,2,2, + 3,3,3,3, 3,3,3,3, +}; + +static const int8_t q1_bit_mask[32] = { + 1, 2, 4, 8, 16, 32, 64, -128, + 1, 2, 4, 8, 16, 32, 64, -128, + 1, 2, 4, 8, 16, 32, 64, -128, + 1, 2, 4, 8, 16, 32, 64, -128, +}; + +static inline uint32_t ggml_load_u32(const void * p) { + return *(const uint32_t *)p; +} + +#if defined(__F16C__) +static inline __m256 ggml_cvt_fp16x4_to_fp32x2(const ggml_half * p) { + __m128i h = _mm_loadu_si64((const void *)p); + __m128 f = _mm_cvtph_ps(h); + return _mm256_insertf128_ps(_mm256_castps128_ps256(f), f, 1); +} +#endif + + + +static inline __m256i ggml_q1_negmask_8cols( + uint32_t qpack, + __m256i byte_sel, + __m256i bit_mask, + __m256i zero) { + const __m256i qsrc = _mm256_set1_epi32((int32_t)qpack); + const __m256i qbyte = _mm256_shuffle_epi8(qsrc, byte_sel); + const __m256i qbit = _mm256_and_si256(qbyte, bit_mask); + // 0xff where q1 bit is zero, 0x00 where q1 bit is set + return _mm256_cmpeq_epi8(qbit, zero); +} + +// xor-sub pairs: accumulate in int16 (safe: 8*2*127=2032 < 32767) +static inline __m256i ggml_q1_dotpairs_xorsub_i16( + __m256i negmask, + __m256i yrep, + __m256i ones_8) { + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(yrep, negmask), negmask); + return _mm256_maddubs_epi16(ones_8, sy); +} + +// Bit extraction as 0/1 for positive-sum formulation: dot = 2*pos - total +static inline __m256i ggml_q1_bit01_8cols( + uint32_t qpack, + __m256i byte_sel, + __m256i bit_mask, + __m256i ones_8, + __m256i zero) { + const __m256i qsrc = _mm256_set1_epi32((int32_t) qpack); + const __m256i qbyte = _mm256_shuffle_epi8(qsrc, byte_sel); + const __m256i qbit = _mm256_and_si256(qbyte, bit_mask); + const __m256i is_zero = _mm256_cmpeq_epi8(qbit, zero); + return _mm256_andnot_si256(is_zero, ones_8); +} + +static inline __m256i ggml_q8_totalpairs_i16( + __m256i yrep, + __m256i ones_8) { + return _mm256_maddubs_epi16(ones_8, yrep); +} + +static inline __m256i ggml_q1_pospairs_i16( + __m256i bit01, + __m256i yrep) { + return _mm256_maddubs_epi16(bit01, yrep); +} + +void ggml_gemv_q1_0_8x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX2__) && defined(__FMA__) + assert(n % QK1_0 == 0); + assert(nc % 16 == 0); + assert(nr == 1); + + UNUSED(bs); + UNUSED(nr); + + const int nb = n / QK1_0; + + const block_q1_0x8 * GGML_RESTRICT x = (const block_q1_0x8 *) vx; + const block_q8_0 * GGML_RESTRICT y = (const block_q8_0 *) vy; + + const __m256i byte_sel = _mm256_loadu_si256((const __m256i *) q1_byte_sel); + const __m256i bit_mask = _mm256_loadu_si256((const __m256i *) q1_bit_mask); + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i zero = _mm256_setzero_si256(); + + // 48-column main loop (AVX-512 register file) / 32-column (AVX2) +#if defined(__AVX512VL__) + { + const int ncols48 = nc / 48; + const int nc_tail = nc - ncols48 * 48; + + for (int cx = 0; cx < ncols48; ++cx) { + const block_q1_0x8 * GGML_RESTRICT x0 = x + (size_t)(cx * 6) * nb; + const block_q1_0x8 * GGML_RESTRICT x1 = x0 + nb; + const block_q1_0x8 * GGML_RESTRICT x2 = x1 + nb; + const block_q1_0x8 * GGML_RESTRICT x3 = x2 + nb; + const block_q1_0x8 * GGML_RESTRICT x4 = x3 + nb; + const block_q1_0x8 * GGML_RESTRICT x5 = x4 + nb; + + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + __m256 acc4 = _mm256_setzero_ps(); + __m256 acc5 = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmp0 = _mm256_setzero_ps(); + __m256 tmp1 = _mm256_setzero_ps(); + __m256 tmp2 = _mm256_setzero_ps(); + __m256 tmp3 = _mm256_setzero_ps(); + __m256 tmp4 = _mm256_setzero_ps(); + __m256 tmp5 = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * GGML_RESTRICT yb = &y[l * 4 + sb]; + + __m256i total16 = _mm256_setzero_si256(); + __m256i pos0 = _mm256_setzero_si256(); + __m256i pos1 = _mm256_setzero_si256(); + __m256i pos2 = _mm256_setzero_si256(); + __m256i pos3 = _mm256_setzero_si256(); + __m256i pos4 = _mm256_setzero_si256(); + __m256i pos5 = _mm256_setzero_si256(); + + for (int g = 0; g < 8; ++g) { + const __m256i yrep = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 4)); + + total16 = _mm256_add_epi16(total16, ggml_q8_totalpairs_i16(yrep, ones_8)); + + const uint32_t qpack0 = ggml_load_u32(x0[l].qs + (sb * 8 + g) * 4); + const __m256i bit0 = ggml_q1_bit01_8cols(qpack0, byte_sel, bit_mask, ones_8, zero); + pos0 = _mm256_add_epi16(pos0, ggml_q1_pospairs_i16(bit0, yrep)); + + const uint32_t qpack1 = ggml_load_u32(x1[l].qs + (sb * 8 + g) * 4); + const __m256i bit1 = ggml_q1_bit01_8cols(qpack1, byte_sel, bit_mask, ones_8, zero); + pos1 = _mm256_add_epi16(pos1, ggml_q1_pospairs_i16(bit1, yrep)); + + const uint32_t qpack2 = ggml_load_u32(x2[l].qs + (sb * 8 + g) * 4); + const __m256i bit2 = ggml_q1_bit01_8cols(qpack2, byte_sel, bit_mask, ones_8, zero); + pos2 = _mm256_add_epi16(pos2, ggml_q1_pospairs_i16(bit2, yrep)); + + const uint32_t qpack3 = ggml_load_u32(x3[l].qs + (sb * 8 + g) * 4); + const __m256i bit3 = ggml_q1_bit01_8cols(qpack3, byte_sel, bit_mask, ones_8, zero); + pos3 = _mm256_add_epi16(pos3, ggml_q1_pospairs_i16(bit3, yrep)); + + const uint32_t qpack4 = ggml_load_u32(x4[l].qs + (sb * 8 + g) * 4); + const __m256i bit4 = ggml_q1_bit01_8cols(qpack4, byte_sel, bit_mask, ones_8, zero); + pos4 = _mm256_add_epi16(pos4, ggml_q1_pospairs_i16(bit4, yrep)); + + const uint32_t qpack5 = ggml_load_u32(x5[l].qs + (sb * 8 + g) * 4); + const __m256i bit5 = ggml_q1_bit01_8cols(qpack5, byte_sel, bit_mask, ones_8, zero); + pos5 = _mm256_add_epi16(pos5, ggml_q1_pospairs_i16(bit5, yrep)); + } + + const __m256i dot0_16 = _mm256_sub_epi16(_mm256_add_epi16(pos0, pos0), total16); + const __m256i dot1_16 = _mm256_sub_epi16(_mm256_add_epi16(pos1, pos1), total16); + const __m256i dot2_16 = _mm256_sub_epi16(_mm256_add_epi16(pos2, pos2), total16); + const __m256i dot3_16 = _mm256_sub_epi16(_mm256_add_epi16(pos3, pos3), total16); + const __m256i dot4_16 = _mm256_sub_epi16(_mm256_add_epi16(pos4, pos4), total16); + const __m256i dot5_16 = _mm256_sub_epi16(_mm256_add_epi16(pos5, pos5), total16); + + const __m256 ydv = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + tmp0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot0_16, ones_16)), ydv, tmp0); + tmp1 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot1_16, ones_16)), ydv, tmp1); + tmp2 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot2_16, ones_16)), ydv, tmp2); + tmp3 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot3_16, ones_16)), ydv, tmp3); + tmp4 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot4_16, ones_16)), ydv, tmp4); + tmp5 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot5_16, ones_16)), ydv, tmp5); + } + + const __m256 bdv0 = GGML_F32Cx8_LOAD(x0[l].d); + const __m256 bdv1 = GGML_F32Cx8_LOAD(x1[l].d); + const __m256 bdv2 = GGML_F32Cx8_LOAD(x2[l].d); + const __m256 bdv3 = GGML_F32Cx8_LOAD(x3[l].d); + const __m256 bdv4 = GGML_F32Cx8_LOAD(x4[l].d); + const __m256 bdv5 = GGML_F32Cx8_LOAD(x5[l].d); + + acc0 = _mm256_fmadd_ps(tmp0, bdv0, acc0); + acc1 = _mm256_fmadd_ps(tmp1, bdv1, acc1); + acc2 = _mm256_fmadd_ps(tmp2, bdv2, acc2); + acc3 = _mm256_fmadd_ps(tmp3, bdv3, acc3); + acc4 = _mm256_fmadd_ps(tmp4, bdv4, acc4); + acc5 = _mm256_fmadd_ps(tmp5, bdv5, acc5); + } + + float *sbase = s + cx * 48; + _mm256_storeu_ps(sbase + 0, acc0); + _mm256_storeu_ps(sbase + 8, acc1); + _mm256_storeu_ps(sbase + 16, acc2); + _mm256_storeu_ps(sbase + 24, acc3); + _mm256_storeu_ps(sbase + 32, acc4); + _mm256_storeu_ps(sbase + 40, acc5); + } + + // 32-column tail + if (nc_tail >= 32) { + const block_q1_0x8 * GGML_RESTRICT xa = x + (size_t)(ncols48 * 6) * nb; + const block_q1_0x8 * GGML_RESTRICT xb = xa + nb; + const block_q1_0x8 * GGML_RESTRICT xc = xb + nb; + const block_q1_0x8 * GGML_RESTRICT xd = xc + nb; + + __m256 acca = _mm256_setzero_ps(); + __m256 accb = _mm256_setzero_ps(); + __m256 accc = _mm256_setzero_ps(); + __m256 accd = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmpa = _mm256_setzero_ps(); + __m256 tmpb = _mm256_setzero_ps(); + __m256 tmpc = _mm256_setzero_ps(); + __m256 tmpd = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * GGML_RESTRICT yb = &y[l * 4 + sb]; + + __m256i total16 = _mm256_setzero_si256(); + __m256i posa = _mm256_setzero_si256(); + __m256i posb = _mm256_setzero_si256(); + __m256i posc = _mm256_setzero_si256(); + __m256i posd = _mm256_setzero_si256(); + + for (int g = 0; g < 8; ++g) { + const __m256i yrep = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 4)); + + total16 = _mm256_add_epi16(total16, ggml_q8_totalpairs_i16(yrep, ones_8)); + + const uint32_t qpacka = ggml_load_u32(xa[l].qs + (sb * 8 + g) * 4); + const __m256i bita = ggml_q1_bit01_8cols(qpacka, byte_sel, bit_mask, ones_8, zero); + posa = _mm256_add_epi16(posa, ggml_q1_pospairs_i16(bita, yrep)); + + const uint32_t qpackb = ggml_load_u32(xb[l].qs + (sb * 8 + g) * 4); + const __m256i bitb = ggml_q1_bit01_8cols(qpackb, byte_sel, bit_mask, ones_8, zero); + posb = _mm256_add_epi16(posb, ggml_q1_pospairs_i16(bitb, yrep)); + + const uint32_t qpackc = ggml_load_u32(xc[l].qs + (sb * 8 + g) * 4); + const __m256i bitc = ggml_q1_bit01_8cols(qpackc, byte_sel, bit_mask, ones_8, zero); + posc = _mm256_add_epi16(posc, ggml_q1_pospairs_i16(bitc, yrep)); + + const uint32_t qpackd = ggml_load_u32(xd[l].qs + (sb * 8 + g) * 4); + const __m256i bitd = ggml_q1_bit01_8cols(qpackd, byte_sel, bit_mask, ones_8, zero); + posd = _mm256_add_epi16(posd, ggml_q1_pospairs_i16(bitd, yrep)); + } + + const __m256i dota_16 = _mm256_sub_epi16(_mm256_add_epi16(posa, posa), total16); + const __m256i dotb_16 = _mm256_sub_epi16(_mm256_add_epi16(posb, posb), total16); + const __m256i dotc_16 = _mm256_sub_epi16(_mm256_add_epi16(posc, posc), total16); + const __m256i dotd_16 = _mm256_sub_epi16(_mm256_add_epi16(posd, posd), total16); + + const __m256 ydv = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + tmpa = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dota_16, ones_16)), ydv, tmpa); + tmpb = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dotb_16, ones_16)), ydv, tmpb); + tmpc = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dotc_16, ones_16)), ydv, tmpc); + tmpd = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dotd_16, ones_16)), ydv, tmpd); + } + + const __m256 bdva = GGML_F32Cx8_LOAD(xa[l].d); + const __m256 bdvb = GGML_F32Cx8_LOAD(xb[l].d); + const __m256 bdvc = GGML_F32Cx8_LOAD(xc[l].d); + const __m256 bdvd = GGML_F32Cx8_LOAD(xd[l].d); + + acca = _mm256_fmadd_ps(tmpa, bdva, acca); + accb = _mm256_fmadd_ps(tmpb, bdvb, accb); + accc = _mm256_fmadd_ps(tmpc, bdvc, accc); + accd = _mm256_fmadd_ps(tmpd, bdvd, accd); + } + + float *sbase = s + ncols48 * 48; + _mm256_storeu_ps(sbase + 0, acca); + _mm256_storeu_ps(sbase + 8, accb); + _mm256_storeu_ps(sbase + 16, accc); + _mm256_storeu_ps(sbase + 24, accd); + } + + // 16-column tail + if (nc_tail == 16) { + const block_q1_0x8 * GGML_RESTRICT xa = x + (size_t)(ncols48 * 6) * nb; + const block_q1_0x8 * GGML_RESTRICT xb = xa + nb; + + __m256 acca = _mm256_setzero_ps(); + __m256 accb = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmpa = _mm256_setzero_ps(); + __m256 tmpb = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * GGML_RESTRICT yb = &y[l * 4 + sb]; + + __m256i total16 = _mm256_setzero_si256(); + __m256i posa = _mm256_setzero_si256(); + __m256i posb = _mm256_setzero_si256(); + + for (int g = 0; g < 8; ++g) { + const __m256i yrep = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 4)); + + total16 = _mm256_add_epi16(total16, ggml_q8_totalpairs_i16(yrep, ones_8)); + + const uint32_t qpacka = ggml_load_u32(xa[l].qs + (sb * 8 + g) * 4); + const __m256i bita = ggml_q1_bit01_8cols(qpacka, byte_sel, bit_mask, ones_8, zero); + posa = _mm256_add_epi16(posa, ggml_q1_pospairs_i16(bita, yrep)); + + const uint32_t qpackb = ggml_load_u32(xb[l].qs + (sb * 8 + g) * 4); + const __m256i bitb = ggml_q1_bit01_8cols(qpackb, byte_sel, bit_mask, ones_8, zero); + posb = _mm256_add_epi16(posb, ggml_q1_pospairs_i16(bitb, yrep)); + } + + const __m256i dota_16 = _mm256_sub_epi16(_mm256_add_epi16(posa, posa), total16); + const __m256i dotb_16 = _mm256_sub_epi16(_mm256_add_epi16(posb, posb), total16); + + const __m256 ydv = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + tmpa = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dota_16, ones_16)), ydv, tmpa); + tmpb = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dotb_16, ones_16)), ydv, tmpb); + } + + const __m256 bdva = GGML_F32Cx8_LOAD(xa[l].d); + const __m256 bdvb = GGML_F32Cx8_LOAD(xb[l].d); + + acca = _mm256_fmadd_ps(tmpa, bdva, acca); + accb = _mm256_fmadd_ps(tmpb, bdvb, accb); + } + + float *sbase = s + ncols48 * 48; + _mm256_storeu_ps(sbase, acca); + _mm256_storeu_ps(sbase + 8, accb); + } } +#else + { + const int ncols32 = nc / 32; + const int nc_tail = nc - ncols32 * 32; -void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { -#if defined( __AVX2__ ) || defined( __AVX512F__ ) - { - assert (n % QK1_0 == 0); - assert (nr % 4 == 0); - assert (nc % 8 == 0); + for (int cx = 0; cx < ncols32; ++cx) { + const block_q1_0x8 * GGML_RESTRICT x0 = x + (size_t)(cx * 4) * nb; + const block_q1_0x8 * GGML_RESTRICT x1 = x0 + nb; + const block_q1_0x8 * GGML_RESTRICT x2 = x1 + nb; + const block_q1_0x8 * GGML_RESTRICT x3 = x2 + nb; + + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmp0 = _mm256_setzero_ps(); + __m256 tmp1 = _mm256_setzero_ps(); + __m256 tmp2 = _mm256_setzero_ps(); + __m256 tmp3 = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * GGML_RESTRICT yb = &y[l * 4 + sb]; - UNUSED(bs); + __m256i total16 = _mm256_setzero_si256(); + __m256i pos0 = _mm256_setzero_si256(); + __m256i pos1 = _mm256_setzero_si256(); + __m256i pos2 = _mm256_setzero_si256(); + __m256i pos3 = _mm256_setzero_si256(); - const int nb = n / QK1_0; - const int nb_q8_0 = n / QK8_0; - const int ncols8 = nc / 8; - const int nrows4 = nr / 4; + for (int g = 0; g < 8; ++g) { + const __m256i yrep = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 4)); - const __m256i ones_8 = _mm256_set1_epi8(1); - const __m256i ones_16 = _mm256_set1_epi16(1); - const __m256i zero = _mm256_setzero_si256(); + total16 = _mm256_add_epi16(total16, ggml_q8_totalpairs_i16(yrep, ones_8)); - const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + const uint32_t qpack0 = ggml_load_u32(x0[l].qs + (sb * 8 + g) * 4); + const __m256i bit0 = ggml_q1_bit01_8cols(qpack0, byte_sel, bit_mask, ones_8, zero); + pos0 = _mm256_add_epi16(pos0, ggml_q1_pospairs_i16(bit0, yrep)); - for (int y = 0; y < nrows4; ++y) { - const block_q8_0x4 * a_ptr = (const block_q8_0x4 *)vy + (y * nb_q8_0); + const uint32_t qpack1 = ggml_load_u32(x1[l].qs + (sb * 8 + g) * 4); + const __m256i bit1 = ggml_q1_bit01_8cols(qpack1, byte_sel, bit_mask, ones_8, zero); + pos1 = _mm256_add_epi16(pos1, ggml_q1_pospairs_i16(bit1, yrep)); - for (int row_base = 0; row_base < 4; row_base += 2) { - for (int x8 = 0; x8 < ncols8; ++x8) { - const block_q1_0x8 * b_ptr = vx_bi + (size_t)x8 * nb; - float * s_row0 = s + (y * 4 + row_base + 0) * bs + x8 * 8; - float * s_row1 = s + (y * 4 + row_base + 1) * bs + x8 * 8; + const uint32_t qpack2 = ggml_load_u32(x2[l].qs + (sb * 8 + g) * 4); + const __m256i bit2 = ggml_q1_bit01_8cols(qpack2, byte_sel, bit_mask, ones_8, zero); + pos2 = _mm256_add_epi16(pos2, ggml_q1_pospairs_i16(bit2, yrep)); + + const uint32_t qpack3 = ggml_load_u32(x3[l].qs + (sb * 8 + g) * 4); + const __m256i bit3 = ggml_q1_bit01_8cols(qpack3, byte_sel, bit_mask, ones_8, zero); + pos3 = _mm256_add_epi16(pos3, ggml_q1_pospairs_i16(bit3, yrep)); + } - GGML_GEMM_Q1_0_4COL(1, 2, 4, 8, 0, 1, 2, 3, 0) - GGML_GEMM_Q1_0_4COL(16, 32, 64, -128, 4, 5, 6, 7, 4) + const __m256i dot0_16 = _mm256_sub_epi16(_mm256_add_epi16(pos0, pos0), total16); + const __m256i dot1_16 = _mm256_sub_epi16(_mm256_add_epi16(pos1, pos1), total16); + const __m256i dot2_16 = _mm256_sub_epi16(_mm256_add_epi16(pos2, pos2), total16); + const __m256i dot3_16 = _mm256_sub_epi16(_mm256_add_epi16(pos3, pos3), total16); + + const __m256 ydv = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + tmp0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot0_16, ones_16)), ydv, tmp0); + tmp1 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot1_16, ones_16)), ydv, tmp1); + tmp2 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot2_16, ones_16)), ydv, tmp2); + tmp3 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dot3_16, ones_16)), ydv, tmp3); } + + const __m256 bdv0 = GGML_F32Cx8_LOAD(x0[l].d); + const __m256 bdv1 = GGML_F32Cx8_LOAD(x1[l].d); + const __m256 bdv2 = GGML_F32Cx8_LOAD(x2[l].d); + const __m256 bdv3 = GGML_F32Cx8_LOAD(x3[l].d); + + acc0 = _mm256_fmadd_ps(tmp0, bdv0, acc0); + acc1 = _mm256_fmadd_ps(tmp1, bdv1, acc1); + acc2 = _mm256_fmadd_ps(tmp2, bdv2, acc2); + acc3 = _mm256_fmadd_ps(tmp3, bdv3, acc3); } + + float *sbase = s + cx * 32; + _mm256_storeu_ps(sbase + 0, acc0); + _mm256_storeu_ps(sbase + 8, acc1); + _mm256_storeu_ps(sbase + 16, acc2); + _mm256_storeu_ps(sbase + 24, acc3); } - return; + // 16-column tail + if (nc_tail >= 16) { + const block_q1_0x8 * GGML_RESTRICT xa = x + (size_t)(ncols32 * 4) * nb; + const block_q1_0x8 * GGML_RESTRICT xb = xa + nb; + + __m256 acca = _mm256_setzero_ps(); + __m256 accb = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmpa = _mm256_setzero_ps(); + __m256 tmpb = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0 * GGML_RESTRICT yb = &y[l * 4 + sb]; + + __m256i total16 = _mm256_setzero_si256(); + __m256i posa = _mm256_setzero_si256(); + __m256i posb = _mm256_setzero_si256(); + + for (int g = 0; g < 8; ++g) { + const __m256i yrep = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 4)); + + total16 = _mm256_add_epi16(total16, ggml_q8_totalpairs_i16(yrep, ones_8)); + + const uint32_t qpacka = ggml_load_u32(xa[l].qs + (sb * 8 + g) * 4); + const __m256i bita = ggml_q1_bit01_8cols(qpacka, byte_sel, bit_mask, ones_8, zero); + posa = _mm256_add_epi16(posa, ggml_q1_pospairs_i16(bita, yrep)); + + const uint32_t qpackb = ggml_load_u32(xb[l].qs + (sb * 8 + g) * 4); + const __m256i bitb = ggml_q1_bit01_8cols(qpackb, byte_sel, bit_mask, ones_8, zero); + posb = _mm256_add_epi16(posb, ggml_q1_pospairs_i16(bitb, yrep)); + } + + const __m256i dota_16 = _mm256_sub_epi16(_mm256_add_epi16(posa, posa), total16); + const __m256i dotb_16 = _mm256_sub_epi16(_mm256_add_epi16(posb, posb), total16); + + const __m256 ydv = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(yb->d)); + tmpa = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dota_16, ones_16)), ydv, tmpa); + tmpb = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(dotb_16, ones_16)), ydv, tmpb); + } + + const __m256 bdva = GGML_F32Cx8_LOAD(xa[l].d); + const __m256 bdvb = GGML_F32Cx8_LOAD(xb[l].d); + + acca = _mm256_fmadd_ps(tmpa, bdva, acca); + accb = _mm256_fmadd_ps(tmpb, bdvb, accb); + } + + _mm256_storeu_ps(s + ncols32 * 32, acca); + _mm256_storeu_ps(s + ncols32 * 32 + 8, accb); + } } -#endif // defined( __AVX2__ ) || defined( __AVX512F__ ) +#endif // __AVX512VL__ + + return; +#else + ggml_gemv_q1_0_8x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} - ggml_gemm_q1_0_8x32_q8_0_generic(n, s, bs, vx, vy, nr, nc); +void ggml_gemm_q1_0_8x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX2__) && defined(__FMA__) + assert(n % QK1_0 == 0); + assert(nr % 4 == 0); + assert(nc % 16 == 0); + + const int nb = n / QK1_0; + const int nb_q8_0 = n / QK8_0; + const int ncols16 = nc / 16; + const int nrows4 = nr / 4; + + const block_q1_0x8 * vx_bi = (const block_q1_0x8 *)vx; + + const __m256i byte_sel = _mm256_loadu_si256((const __m256i *) q1_byte_sel); + const __m256i bit_mask = _mm256_loadu_si256((const __m256i *) q1_bit_mask); + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i zero = _mm256_setzero_si256(); + + for (int y = 0; y < nrows4; ++y) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *)vy + (y * nb_q8_0); + + for (int x = 0; x < ncols16; ++x) { + const block_q1_0x8 * ba = vx_bi + (size_t)(x * 2) * nb; + const block_q1_0x8 * bb = ba + nb; + + __m256 acc0a = _mm256_setzero_ps(); + __m256 acc1a = _mm256_setzero_ps(); + __m256 acc2a = _mm256_setzero_ps(); + __m256 acc3a = _mm256_setzero_ps(); + __m256 acc0b = _mm256_setzero_ps(); + __m256 acc1b = _mm256_setzero_ps(); + __m256 acc2b = _mm256_setzero_ps(); + __m256 acc3b = _mm256_setzero_ps(); + + for (int l = 0; l < nb; ++l) { + __m256 tmp0a = _mm256_setzero_ps(); + __m256 tmp1a = _mm256_setzero_ps(); + __m256 tmp2a = _mm256_setzero_ps(); + __m256 tmp3a = _mm256_setzero_ps(); + __m256 tmp0b = _mm256_setzero_ps(); + __m256 tmp1b = _mm256_setzero_ps(); + __m256 tmp2b = _mm256_setzero_ps(); + __m256 tmp3b = _mm256_setzero_ps(); + + for (int sb = 0; sb < 4; ++sb) { + const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; + + __m256i isum0a16 = _mm256_setzero_si256(); + __m256i isum1a16 = _mm256_setzero_si256(); + __m256i isum2a16 = _mm256_setzero_si256(); + __m256i isum3a16 = _mm256_setzero_si256(); + __m256i isum0b16 = _mm256_setzero_si256(); + __m256i isum1b16 = _mm256_setzero_si256(); + __m256i isum2b16 = _mm256_setzero_si256(); + __m256i isum3b16 = _mm256_setzero_si256(); + + for (int g = 0; g < 8; ++g) { + const uint32_t qpacka = ggml_load_u32(ba[l].qs + (sb * 8 + g) * 4); + const __m256i nega = ggml_q1_negmask_8cols(qpacka, byte_sel, bit_mask, zero); + const uint32_t qpackb = ggml_load_u32(bb[l].qs + (sb * 8 + g) * 4); + const __m256i negb = ggml_q1_negmask_8cols(qpackb, byte_sel, bit_mask, zero); + + const __m256i yrep0 = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 16 + 0 * 4)); + const __m256i yrep1 = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 16 + 1 * 4)); + const __m256i yrep2 = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 16 + 2 * 4)); + const __m256i yrep3 = _mm256_set1_epi32(ggml_load_u32(yb->qs + g * 16 + 3 * 4)); + + isum0a16 = _mm256_add_epi16(isum0a16, ggml_q1_dotpairs_xorsub_i16(nega, yrep0, ones_8)); + isum1a16 = _mm256_add_epi16(isum1a16, ggml_q1_dotpairs_xorsub_i16(nega, yrep1, ones_8)); + isum2a16 = _mm256_add_epi16(isum2a16, ggml_q1_dotpairs_xorsub_i16(nega, yrep2, ones_8)); + isum3a16 = _mm256_add_epi16(isum3a16, ggml_q1_dotpairs_xorsub_i16(nega, yrep3, ones_8)); + + isum0b16 = _mm256_add_epi16(isum0b16, ggml_q1_dotpairs_xorsub_i16(negb, yrep0, ones_8)); + isum1b16 = _mm256_add_epi16(isum1b16, ggml_q1_dotpairs_xorsub_i16(negb, yrep1, ones_8)); + isum2b16 = _mm256_add_epi16(isum2b16, ggml_q1_dotpairs_xorsub_i16(negb, yrep2, ones_8)); + isum3b16 = _mm256_add_epi16(isum3b16, ggml_q1_dotpairs_xorsub_i16(negb, yrep3, ones_8)); + } + + const __m256i isum0a = _mm256_madd_epi16(isum0a16, ones_16); + const __m256i isum1a = _mm256_madd_epi16(isum1a16, ones_16); + const __m256i isum2a = _mm256_madd_epi16(isum2a16, ones_16); + const __m256i isum3a = _mm256_madd_epi16(isum3a16, ones_16); + const __m256i isum0b = _mm256_madd_epi16(isum0b16, ones_16); + const __m256i isum1b = _mm256_madd_epi16(isum1b16, ones_16); + const __m256i isum2b = _mm256_madd_epi16(isum2b16, ones_16); + const __m256i isum3b = _mm256_madd_epi16(isum3b16, ones_16); + + const __m256 yd_all = ggml_cvt_fp16x4_to_fp32x2(yb->d); + const __m256 yd0 = _mm256_permute_ps(yd_all, 0); + const __m256 yd1 = _mm256_permute_ps(yd_all, 85); + const __m256 yd2 = _mm256_permute_ps(yd_all, 170); + const __m256 yd3 = _mm256_permute_ps(yd_all, 255); + tmp0a = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum0a), yd0, tmp0a); + tmp1a = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum1a), yd1, tmp1a); + tmp2a = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum2a), yd2, tmp2a); + tmp3a = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum3a), yd3, tmp3a); + tmp0b = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum0b), yd0, tmp0b); + tmp1b = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum1b), yd1, tmp1b); + tmp2b = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum2b), yd2, tmp2b); + tmp3b = _mm256_fmadd_ps(_mm256_cvtepi32_ps(isum3b), yd3, tmp3b); + } + + const __m256 bdva = GGML_F32Cx8_LOAD(ba[l].d); + const __m256 bdvb = GGML_F32Cx8_LOAD(bb[l].d); + + acc0a = _mm256_fmadd_ps(tmp0a, bdva, acc0a); + acc1a = _mm256_fmadd_ps(tmp1a, bdva, acc1a); + acc2a = _mm256_fmadd_ps(tmp2a, bdva, acc2a); + acc3a = _mm256_fmadd_ps(tmp3a, bdva, acc3a); + acc0b = _mm256_fmadd_ps(tmp0b, bdvb, acc0b); + acc1b = _mm256_fmadd_ps(tmp1b, bdvb, acc1b); + acc2b = _mm256_fmadd_ps(tmp2b, bdvb, acc2b); + acc3b = _mm256_fmadd_ps(tmp3b, bdvb, acc3b); + } + + _mm256_storeu_ps(s + (y * 4 + 0) * bs + x * 16, acc0a); + _mm256_storeu_ps(s + (y * 4 + 1) * bs + x * 16, acc1a); + _mm256_storeu_ps(s + (y * 4 + 2) * bs + x * 16, acc2a); + _mm256_storeu_ps(s + (y * 4 + 3) * bs + x * 16, acc3a); + _mm256_storeu_ps(s + (y * 4 + 0) * bs + x * 16 + 8, acc0b); + _mm256_storeu_ps(s + (y * 4 + 1) * bs + x * 16 + 8, acc1b); + _mm256_storeu_ps(s + (y * 4 + 2) * bs + x * 16 + 8, acc2b); + _mm256_storeu_ps(s + (y * 4 + 3) * bs + x * 16 + 8, acc3b); + } + } + + return; +#else + ggml_gemm_q1_0_8x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif } void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 7b0a0a1a8a09..8c87c603efce 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -208,42 +208,6 @@ void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GG } } -void ggml_quantize_mat_q8_0_4x32_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { - assert(QK8_0 == 32); - assert(k % QK8_0 == 0); - const int nb = k / QK8_0; - - block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; - - // scalar - float srcv[4][QK8_0]; - float id[4]; - - for (int i = 0; i < nb; i++) { - for (int row_iter = 0; row_iter < 4; row_iter++) { - float amax = 0.0f; // absolute max - - for (int j = 0; j < QK8_0; j++) { - srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j]; - amax = MAX(amax, fabsf(srcv[row_iter][j])); - } - - const float d = amax / ((1 << 7) - 1); - id[row_iter] = d ? 1.0f / d : 0.0f; - - y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d); - } - - // Store each row's 32 bytes contiguously - for (int r = 0; r < 4; r++) { - for (int j = 0; j < QK8_0; j++) { - float x0 = srcv[r][j] * id[r]; - y[i].qs[r * QK8_0 + j] = roundf(x0); - } - } - } -} - void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK_K == 256); assert(k % QK_K == 0); @@ -375,12 +339,6 @@ template <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTR ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row); } -template <> void ggml_quantize_mat_t<32, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { - assert(nrow == 4); - UNUSED(nrow); - ggml_quantize_mat_q8_0_4x32(x, vy, n_per_row); -} - #if defined __riscv_zvfh template <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); @@ -926,7 +884,7 @@ void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, } } -void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +void ggml_gemv_q1_0_8x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { assert (n % QK1_0 == 0); assert (nc % 8 == 0); assert (nr == 1); @@ -955,16 +913,28 @@ void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, for (int sb = 0; sb < 4; ++sb) { const block_q8_0 * yb = &a_ptr[l * 4 + sb]; const float dy = GGML_CPU_FP16_TO_FP32(yb->d); - - const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; const int8_t * y = yb->qs; - for (int c = 0; c < 8; ++c) { - int sumi = 0; - for (int i = 0; i < QK8_0; ++i) { - sumi += ((qs[i] >> c) & 1) ? y[i] : -y[i]; + for (int g = 0; g < 8; ++g) { + const uint8_t * qs_row = (const uint8_t *)b_ptr[l].qs + (sb * 8 + g) * 4; + + for (int pair = 0; pair < 4; ++pair) { + uint8_t byte = qs_row[pair]; + int col_even = 2 * pair; + int col_odd = 2 * pair + 1; + int sumi_even = 0; + int sumi_odd = 0; + for (int t = 0; t < 4; ++t) { + int k = g * 4 + t; + int yv = y[k]; + if ((byte >> t) & 1) sumi_even += yv; + else sumi_even -= yv; + if ((byte >> (t + 4)) & 1) sumi_odd += yv; + else sumi_odd -= yv; + } + block_acc[col_even] += dy * (float)sumi_even; + block_acc[col_odd] += dy * (float)sumi_odd; } - block_acc[c] += dy * (float)sumi; } } @@ -972,9 +942,9 @@ void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, acc[c] += bd[c] * block_acc[c]; } } - + static_assert(sizeof(acc) == 32); - memcpy(s + x*8, acc, sizeof(acc)); + memcpy(s + x * 8, acc, sizeof(acc)); } } @@ -1913,7 +1883,7 @@ void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, } } -void ggml_gemm_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +void ggml_gemm_q1_0_8x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { assert (n % QK1_0 == 0); assert (nr % 4 == 0); assert (nc % 8 == 0); @@ -1943,18 +1913,31 @@ void ggml_gemm_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, for (int sb = 0; sb < 4; ++sb) { const block_q8_0x4 * yb = &a_ptr[l * 4 + sb]; - const uint8_t * qs = (const uint8_t *)b_ptr[l].qs + sb * 32; for (int r = 0; r < 2; ++r) { const float dy = GGML_CPU_FP16_TO_FP32(yb->d[row_base + r]); - - for (int c = 0; c < 8; ++c) { - int sumi = 0; - for (int i = 0; i < QK8_0; ++i) { - const int8_t y_val = yb->qs[(row_base + r) * 32 + i]; - sumi += ((qs[i] >> c) & 1) ? y_val : -y_val; + const int row = row_base + r; + + for (int g = 0; g < 8; ++g) { + const uint8_t * qs_row = (const uint8_t *)b_ptr[l].qs + (sb * 8 + g) * 4; + const int8_t * y = yb->qs + g * 16 + row * 4; + + for (int pair = 0; pair < 4; ++pair) { + uint8_t byte = qs_row[pair]; + int col_even = 2 * pair; + int col_odd = 2 * pair + 1; + int sumi_even = 0; + int sumi_odd = 0; + for (int t = 0; t < 4; ++t) { + int yv = y[t]; + if ((byte >> t) & 1) sumi_even += yv; + else sumi_even -= yv; + if ((byte >> (t + 4)) & 1) sumi_odd += yv; + else sumi_odd -= yv; + } + block_acc[r][col_even] += dy * (float)sumi_even; + block_acc[r][col_odd] += dy * (float)sumi_odd; } - block_acc[r][c] += dy * (float)sumi; } } } @@ -2966,23 +2949,30 @@ static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_in return out; } -static block_q1_0x8 make_block_q1_0x8(block_q1_0 * in, unsigned int blck_size_interleave) { +static block_q1_0x8 make_block_q1_0x8(block_q1_0 * in) { block_q1_0x8 out; - GGML_ASSERT(blck_size_interleave == 8); - for (int i = 0; i < 8; ++i) { out.d[i] = in[i].d; } + memset(out.qs, 0, sizeof(out.qs)); + for (int sb = 0; sb < 4; ++sb) { - for (int i = 0; i < 32; ++i) { - uint8_t byte = 0; - for (int c = 0; c < 8; ++c) { - uint8_t src = in[c].qs[sb * 4 + i / 8]; - byte |= ((src >> (i % 8)) & 1) << c; + for (int g = 0; g < 8; ++g) { + for (int pair = 0; pair < 4; ++pair) { + uint8_t byte = 0; + for (int t = 0; t < 4; ++t) { + int k = g * 4 + t; + int col_even = 2 * pair; + int col_odd = 2 * pair + 1; + uint8_t src_even = in[col_even].qs[sb * 4 + k / 8]; + uint8_t src_odd = in[col_odd].qs[sb * 4 + k / 8]; + if ((src_even >> (k % 8)) & 1) byte |= (1 << t); + if ((src_odd >> (k % 8)) & 1) byte |= (1 << (t + 4)); + } + out.qs[(sb * 8 + g) * 4 + pair] = byte; } - out.qs[sb * 32 + i] = byte; } } @@ -3659,8 +3649,8 @@ static int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block } static int repack_q1_0_to_q1_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { + UNUSED(interleave_block); GGML_ASSERT(t->type == GGML_TYPE_Q1_0); - GGML_ASSERT(interleave_block == 8); constexpr int nrows_interleaved = 8; block_q1_0x8 * dst = (block_q1_0x8 *) t->data; @@ -3680,7 +3670,7 @@ static int repack_q1_0_to_q1_0_8_bl(struct ggml_tensor * t, int interleave_block for (int i = 0; i < nrows_interleaved; ++i) { dst_tmp[i] = src[x + i * nblocks]; } - *dst++ = make_block_q1_0x8(dst_tmp, interleave_block); + *dst++ = make_block_q1_0x8(dst_tmp); } src += nrows_interleaved * nblocks; } @@ -4145,8 +4135,8 @@ template <> int repack(struct ggml_tensor * t, const void * da return repack_q8_0_to_q8_0_4_bl(t, 8, data, data_size); } -template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { - return repack_q1_0_to_q1_0_8_bl(t, 8, data, data_size); +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_8_bl(t, 0, data, data_size); } #if defined __riscv_zvfh @@ -4246,8 +4236,8 @@ template <> void gemv(int n, float * s, size_t ggml_gemv_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { - ggml_gemv_q1_0_8x32_q8_0(n, s, bs, vx, vy, nr, nc); +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_8x4_q8_0(n, s, bs, vx, vy, nr, nc); } #if defined __riscv_zvfh @@ -4347,8 +4337,8 @@ template <> void gemm(int n, float * s, size_t ggml_gemm_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { - ggml_gemm_q1_0_8x32_q8_0(n, s, bs, vx, vy, nr, nc); +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_8x4_q8_0(n, s, bs, vx, vy, nr, nc); } #if defined __riscv_zvfh @@ -4782,7 +4772,7 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons static const ggml::cpu::repack::tensor_traits q8_0_4x8_q8_0; // instance for Q1_0 - static const ggml::cpu::repack::tensor_traits q1_0_8x32_q8_0; + static const ggml::cpu::repack::tensor_traits q1_0_8x4_q8_0; // instances for RISC-V // @@ -4825,8 +4815,8 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons } } else if (cur->type == GGML_TYPE_Q1_0) { if (ggml_cpu_has_avx2()) { - if (cur->ne[1] % 8 == 0) { - return &q1_0_8x32_q8_0; + if (cur->ne[1] % 16 == 0) { + return &q1_0_8x4_q8_0; } } } else if (cur->type == GGML_TYPE_Q4_K) { diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index 5db7ca8269ff..4307cc4f2cbb 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -35,7 +35,7 @@ static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, "wrong static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong block<8,4> size/padding"); static_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding"); static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding"); -static_assert(sizeof(block<1, 8>) == 8 * sizeof(ggml_half) + QK1_0, "wrong block<1,8> size/padding"); +static_assert(sizeof(block<1, 8>) == 8 * sizeof(ggml_half) + QK1_0, "wrong block_q1_0x8 size"); using block_q1_0x8 = block<1, 8>; using block_q4_0x4 = block<4, 4>; @@ -144,7 +144,6 @@ extern "C" { void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); -void ggml_quantize_mat_q8_0_4x32(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -163,7 +162,7 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); -void ggml_gemv_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_8x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -180,7 +179,7 @@ void ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); -void ggml_gemm_q1_0_8x32_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_8x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); @@ -199,7 +198,6 @@ void ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v // Native implementations void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); -void ggml_quantize_mat_q8_0_4x32_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -218,7 +216,7 @@ void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); -void ggml_gemv_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_8x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -235,7 +233,7 @@ void ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); -void ggml_gemm_q1_0_8x32_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_8x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);