diff --git a/gallery/index.yaml b/gallery/index.yaml index f3ae01885450..da4570bd626d 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -376,6 +376,207 @@ - filename: llama-cpp/mmproj/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/mmproj-F32.gguf sha256: 32f7ea0600c07272547da401d460f8abbd980f3a57b69d6df87be0e2505e0b9c uri: https://huggingface.co/Jackrong/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/resolve/main/mmproj-F32.gguf +# DFlash speculative-decoding pairs (upstream llama.cpp `draft-dflash`). +# Each entry ships a full target model plus a small block-diffusion drafter +# (z-lab DFlash, converted with upstream convert_hf_to_gguf.py, GGUF arch +# `dflash`). Requires flash attention (enabled below). GPU recommended. +- name: "qwen3-4b-dflash" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3-4B + - https://huggingface.co/unsloth/Qwen3-4B-GGUF + - https://huggingface.co/z-lab/Qwen3-4B-DFlash + - https://huggingface.co/lym00/Qwen3-4B-DFlash-GGUF-Test + description: | + Qwen3-4B paired with its DFlash block-diffusion drafter for speculative decoding on the llama.cpp backend. This is the canonical DFlash pairing documented upstream (`z-lab/Qwen3-4B-DFlash` + `Qwen/Qwen3-4B`). + + DFlash produces a whole block of draft tokens in a single forward pass and injects the target model's hidden states into the drafter's attention, which keeps the drafter tiny while making drafting GPU-friendly. The Q4_K_M file carries the full Qwen3-4B target; the ~0.5 GB Q8_0 drafter (`draft-dflash`) accelerates generation without changing the target's outputs. The drafter is not a standalone chat model: it only runs paired with the target, which is why both are bundled here. + + Flash attention is required for DFlash and is enabled in this config. A GPU is recommended. + + License: Apache 2.0 (Qwen3-4B target) / MIT (z-lab DFlash drafter). + license: "apache-2.0" + tags: + - llm + - gguf + - gpu + - qwen + - dflash + - speculative + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + overrides: + backend: llama-cpp + flash_attention: "on" + draft_model: llama-cpp/models/Qwen3-4B-DFlash-GGUF-Test/Qwen3-4B-DFlash-q8_0.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - spec_type:draft-dflash + - spec_n_max:15 + parameters: + model: llama-cpp/models/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf + sha256: f6f851777709861056efcdad3af01da38b31223a3ba26e61a4f8bf3a2195813a + uri: https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/main/Qwen3-4B-Q4_K_M.gguf + - filename: llama-cpp/models/Qwen3-4B-DFlash-GGUF-Test/Qwen3-4B-DFlash-q8_0.gguf + sha256: 5ecf02bb269fc42277f43961794387fea11ecc367ea2d99e86b2b71cc249aff6 + uri: https://huggingface.co/lym00/Qwen3-4B-DFlash-GGUF-Test/resolve/main/Qwen3-4B-DFlash-q8_0.gguf +- name: "qwen3.5-9b-dflash" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.5-9B + - https://huggingface.co/unsloth/Qwen3.5-9B-GGUF + - https://huggingface.co/z-lab/Qwen3.5-9B-DFlash + - https://huggingface.co/lym00/Qwen3.5-9B-DFlash-GGUF-Test + description: | + Qwen3.5-9B paired with its DFlash block-diffusion drafter for speculative decoding on the llama.cpp backend. + + DFlash produces a whole block of draft tokens in a single forward pass and injects the target model's hidden states into the drafter's attention, which keeps the drafter tiny while making drafting GPU-friendly. The Q4_K_M file carries the full Qwen3.5-9B target; the ~1 GB Q8_0 drafter (`draft-dflash`) accelerates generation without changing the target's outputs. The drafter is not a standalone chat model: it only runs paired with the target, which is why both are bundled here. + + Flash attention is required for DFlash and is enabled in this config. A GPU is recommended. + + License: Apache 2.0 (Qwen3.5-9B target) / MIT (z-lab DFlash drafter). + license: "apache-2.0" + tags: + - llm + - gguf + - gpu + - qwen + - dflash + - speculative + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + overrides: + backend: llama-cpp + flash_attention: "on" + draft_model: llama-cpp/models/Qwen3.5-9B-DFlash-GGUF-Test/Qwen3.5-9B-DFlash-q8_0.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - spec_type:draft-dflash + - spec_n_max:15 + parameters: + model: llama-cpp/models/Qwen3.5-9B-GGUF/Qwen3.5-9B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3.5-9B-GGUF/Qwen3.5-9B-Q4_K_M.gguf + sha256: 03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8 + uri: https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf + - filename: llama-cpp/models/Qwen3.5-9B-DFlash-GGUF-Test/Qwen3.5-9B-DFlash-q8_0.gguf + sha256: 4f76ecff951dd371eea318be9ca60b4dd0800a677c11e14c092fab3358251dc1 + uri: https://huggingface.co/lym00/Qwen3.5-9B-DFlash-GGUF-Test/resolve/main/Qwen3.5-9B-DFlash-q8_0.gguf +- name: "qwen3.6-27b-dflash" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.6-27B + - https://huggingface.co/unsloth/Qwen3.6-27B-GGUF + - https://huggingface.co/z-lab/Qwen3.6-27B-DFlash + - https://huggingface.co/Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp + description: | + Qwen3.6-27B (dense) paired with its DFlash block-diffusion drafter for speculative decoding on the llama.cpp backend. DFlash gives its largest speedups on dense targets like this one. + + DFlash produces a whole block of draft tokens in a single forward pass and injects the target model's hidden states into the drafter's attention, which keeps the drafter tiny while making drafting GPU-friendly. The Q4_K_M file carries the full Qwen3.6-27B target; the ~1.8 GB Q8_0 drafter (`draft-dflash`) accelerates generation without changing the target's outputs. The drafter is not a standalone chat model: it only runs paired with the target, which is why both are bundled here. + + Flash attention is required for DFlash and is enabled in this config. A GPU is recommended. + + License: Apache 2.0. + license: "apache-2.0" + tags: + - llm + - gguf + - gpu + - qwen + - dflash + - speculative + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3.6/Figures/qwen3.6_27b_score.png + overrides: + backend: llama-cpp + flash_attention: "on" + draft_model: llama-cpp/models/Qwen3.6-27B-DFlash-GGUF-llama.cpp/Qwen3.6-27B-DFlash-Q8_0.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - spec_type:draft-dflash + - spec_n_max:15 + parameters: + model: llama-cpp/models/Qwen3.6-27B-GGUF/Qwen3.6-27B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3.6-27B-GGUF/Qwen3.6-27B-Q4_K_M.gguf + sha256: 5ed60d0af4650a854b1755bd392f9aef4872643dc25a254bc68043fa638392a0 + uri: https://huggingface.co/unsloth/Qwen3.6-27B-GGUF/resolve/main/Qwen3.6-27B-Q4_K_M.gguf + - filename: llama-cpp/models/Qwen3.6-27B-DFlash-GGUF-llama.cpp/Qwen3.6-27B-DFlash-Q8_0.gguf + sha256: 23b6c8ebcc51b3b4107709342fd2960167e88397af36e394923b8d5895ddf7ea + uri: https://huggingface.co/Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp/resolve/main/Qwen3.6-27B-DFlash-Q8_0.gguf +- name: "qwen3.6-35b-a3b-dflash" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.6-35B-A3B + - https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF + - https://huggingface.co/z-lab/Qwen3.6-35B-A3B-DFlash + - https://huggingface.co/Alittlehammmer/Qwen3.6-35B-A3B-DFlash-GGUF-llama.cpp + description: | + Qwen3.6-35B-A3B (Mixture-of-Experts, ~3B active per token) paired with its DFlash block-diffusion drafter for speculative decoding on the llama.cpp backend. DFlash speedups on MoE targets are smaller than on dense models, but still useful. + + DFlash produces a whole block of draft tokens in a single forward pass and injects the target model's hidden states into the drafter's attention, which keeps the drafter tiny while making drafting GPU-friendly. The UD-Q4_K_M file carries the full Qwen3.6-35B-A3B target; the ~0.4 GB Q8_0 drafter (`draft-dflash`) accelerates generation without changing the target's outputs. The drafter is not a standalone chat model: it only runs paired with the target, which is why both are bundled here. + + Flash attention is required for DFlash and is enabled in this config. A GPU is recommended. + + License: Apache 2.0. + license: "apache-2.0" + tags: + - llm + - gguf + - gpu + - qwen + - moe + - dflash + - speculative + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3.6/Figures/qwen3.6_35b_a3b_score.png + overrides: + backend: llama-cpp + flash_attention: "on" + draft_model: llama-cpp/models/Qwen3.6-35B-A3B-DFlash-GGUF-llama.cpp/Qwen3.6-35B-A3B-DFlash-Q8_0.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - spec_type:draft-dflash + - spec_n_max:15 + parameters: + model: llama-cpp/models/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf + sha256: ac0e2c1189e055faa36eff361580e79c5bd6f8e76bffb4ce547f167d53e31a61 + uri: https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf + - filename: llama-cpp/models/Qwen3.6-35B-A3B-DFlash-GGUF-llama.cpp/Qwen3.6-35B-A3B-DFlash-Q8_0.gguf + sha256: 724ac8a102763e0d896cd49bf558fdf5bfdb146ec9f7097d18b035e4a95eac3f + uri: https://huggingface.co/Alittlehammmer/Qwen3.6-35B-A3B-DFlash-GGUF-llama.cpp/resolve/main/Qwen3.6-35B-A3B-DFlash-Q8_0.gguf - name: "kimi-k2.7-code" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: