diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..d3ca5d916e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,6 +43,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if 1.0.4", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -131,7 +145,7 @@ checksum = "5d0a66767aaf7d483c556386fb68ca2fba9347684d8bb17a4bd8b755851870f7" dependencies = [ "arrayvec", "aws-lc-rs", - "base64", + "base64 0.22.1", "byteorder", "minicbor", "rustls-pki-types", @@ -402,7 +416,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" dependencies = [ - "base64", + "base64 0.22.1", "http", "log", "rustls", @@ -473,7 +487,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "axum-macros", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -549,6 +563,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -567,6 +587,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bip39" version = "2.2.2" @@ -765,7 +791,7 @@ name = "buzz-acp" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "buzz-core", "buzz-persona", "buzz-sdk", @@ -825,7 +851,7 @@ dependencies = [ "arc-swap", "async-trait", "axum", - "base64", + "base64 0.22.1", "getrandom 0.4.3", "hex", "nix 0.31.3", @@ -884,7 +910,7 @@ name = "buzz-cli" version = "0.1.0" dependencies = [ "axum", - "base64", + "base64 0.22.1", "buzz-core", "buzz-persona", "buzz-sdk", @@ -925,7 +951,7 @@ dependencies = [ name = "buzz-core" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "hmac 0.13.0", @@ -965,7 +991,7 @@ dependencies = [ name = "buzz-dev-mcp" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "buzz-cli", "buzz-core", "git-credential-nostr", @@ -1092,7 +1118,7 @@ dependencies = [ "appattest", "async-trait", "axum", - "base64", + "base64 0.22.1", "byteorder", "chrono", "getrandom 0.4.3", @@ -1127,7 +1153,7 @@ dependencies = [ "async-compression", "async-trait", "axum", - "base64", + "base64 0.22.1", "buzz-audit", "buzz-auth", "buzz-conformance", @@ -1237,7 +1263,7 @@ name = "buzz-test-client" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "buzz-core", "buzz-sdk", "buzz-ws-client", @@ -1262,6 +1288,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "ort", + "ort-sys", + "rand 0.10.1", + "sentencepiece-model", + "serde", + "serde_json", + "sherpa-onnx", + "tokenizers", +] + [[package]] name = "buzz-workflow" version = "0.1.0" @@ -1329,6 +1369,26 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "castaway" version = "0.2.4" @@ -1577,6 +1637,7 @@ dependencies = [ "itoa", "rustversion", "ryu", + "serde", "static_assertions", ] @@ -2137,6 +2198,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -2174,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -2626,6 +2696,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etcetera" version = "0.11.0" @@ -2693,6 +2769,17 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-srgb8" version = "1.0.0" @@ -3084,7 +3171,7 @@ dependencies = [ name = "git-credential-nostr" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "nostr", "serde_json", "zeroize", @@ -3094,7 +3181,7 @@ dependencies = [ name = "git-sign-nostr" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "libc", @@ -3269,7 +3356,7 @@ version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f89305dc8fe34e165eaf0eb12b6e294e12381d9df9a431bcc52a5809bab4319" dependencies = [ - "base64", + "base64 0.22.1", "bon", "bytes", "futures", @@ -3575,7 +3662,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -4359,6 +4446,39 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.117", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loom" version = "0.7.2" @@ -4418,6 +4538,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "matchers" version = "0.2.0" @@ -4433,6 +4569,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-async" version = "0.2.11" @@ -4529,7 +4675,7 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292 dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "crypto_box", "ed25519-dalek", @@ -4542,7 +4688,7 @@ dependencies = [ "mesh-llm-types", "model-artifact", "nostr-sdk", - "prost", + "prost 0.14.3", "rand 0.10.1", "rustls", "serde", @@ -4631,7 +4777,7 @@ dependencies = [ "argon2", "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "chacha20poly1305", "chrono", @@ -4681,7 +4827,7 @@ dependencies = [ "opentelemetry 0.31.0", "opentelemetry-otlp 0.31.1", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "rand 0.10.1", "regex-lite", "reqwest 0.12.28", @@ -4721,7 +4867,7 @@ version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292be688ac0261129bcbab0e7b9132" dependencies = [ "argon2", - "base64", + "base64 0.22.1", "chacha20poly1305", "chrono", "crypto_box", @@ -4769,8 +4915,8 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292 dependencies = [ "anyhow", "async-trait", - "prost", - "prost-build", + "prost 0.14.3", + "prost-build 0.14.3", "protoc-bin-vendored", "rmcp", "schemars", @@ -4805,7 +4951,7 @@ dependencies = [ "anyhow", "hex", "iroh", - "prost", + "prost 0.14.3", "serde_json", "sha2 0.10.9", ] @@ -4936,7 +5082,7 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" dependencies = [ - "base64", + "base64 0.22.1", "evmap", "http-body-util", "hyper", @@ -4974,6 +5120,28 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if 1.0.4", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "mime" version = "0.3.17" @@ -5119,6 +5287,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "more-asserts" version = "0.3.1" @@ -5225,6 +5415,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -5461,7 +5666,7 @@ version = "0.44.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e826dd648489de2c5b293920e20b92932ef820302007c1987c758d4d06eeb2cf" dependencies = [ - "base64", + "base64 0.22.1", "bech32", "bip39", "bitcoin_hashes", @@ -5932,7 +6137,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto 0.31.0", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "reqwest 0.12.28", "thiserror 2.0.18", ] @@ -5947,7 +6152,7 @@ dependencies = [ "opentelemetry 0.32.0", "opentelemetry-proto 0.32.0", "opentelemetry_sdk 0.32.1", - "prost", + "prost 0.14.3", "thiserror 2.0.18", "tokio", "tonic", @@ -5960,11 +6165,11 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "base64", + "base64 0.22.1", "const-hex", "opentelemetry 0.31.0", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "serde", "serde_json", "tonic", @@ -5979,7 +6184,7 @@ checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry 0.32.0", "opentelemetry_sdk 0.32.1", - "prost", + "prost 0.14.3", "tonic", "tonic-prost", ] @@ -6061,6 +6266,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -6242,6 +6465,16 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -6374,7 +6607,7 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ - "base64", + "base64 0.22.1", "indexmap", "quick-xml 0.39.4", "serde", @@ -6440,13 +6673,22 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb3713e4977408279158444a18c1a01ac9bf2e7eaf1fbfd1a19ac9cd18d90721" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "derive_more", "hyper-util", @@ -6616,6 +6858,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.3" @@ -6623,7 +6875,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.3", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.117", + "tempfile", ] [[package]] @@ -6636,15 +6908,28 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", - "prost", - "prost-types", + "prost 0.14.3", + "prost-types 0.14.3", "regex", "syn 2.0.117", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost-derive" version = "0.14.3" @@ -6658,13 +6943,35 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-reflect" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" +dependencies = [ + "logos", + "miette", + "once_cell", + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "prost", + "prost 0.14.3", ] [[package]] @@ -6731,6 +7038,33 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "protox" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" +dependencies = [ + "bytes", + "miette", + "prost 0.13.5", + "prost-reflect", + "prost-types 0.13.5", + "protox-parse", + "thiserror 1.0.69", +] + +[[package]] +name = "protox-parse" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" +dependencies = [ + "logos", + "miette", + "prost-types 0.13.5", + "thiserror 1.0.69", +] + [[package]] name = "pulldown-cmark" version = "0.13.4" @@ -7036,7 +7370,7 @@ dependencies = [ "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7099,7 +7433,7 @@ dependencies = [ "strum", "time", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7111,6 +7445,43 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redb" version = "3.1.3" @@ -7232,7 +7603,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -7280,7 +7651,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -7370,7 +7741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", @@ -7438,7 +7809,7 @@ dependencies = [ "async-trait", "aws-creds", "aws-region", - "base64", + "base64 0.22.1", "bytes", "cfg-if 1.0.4", "futures-util", @@ -7839,6 +8210,18 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "sentencepiece-model" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b87bf750a8322c3236d7aa63c1f4a6862187d00d2d8b038e1dfe263bfe43ec" +dependencies = [ + "miette", + "prost 0.13.5", + "prost-build 0.13.5", + "protox", +] + [[package]] name = "serde" version = "1.0.228" @@ -8049,6 +8432,26 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b142d3f255cb4e4b7808ea25869db6f5714e0a3550da355234483b4db552055" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.4" +dependencies = [ + "bzip2", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -8184,8 +8587,8 @@ name = "skippy-protocol" version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292be688ac0261129bcbab0e7b9132" dependencies = [ - "prost", - "prost-build", + "prost 0.14.3", + "prost-build 0.14.3", "protoc-bin-vendored", "serde", ] @@ -8212,7 +8615,7 @@ dependencies = [ "anyhow", "async-trait", "axum", - "base64", + "base64 0.22.1", "blake3", "clap", "futures-util", @@ -8319,6 +8722,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sprig" version = "0.1.0" @@ -8347,7 +8762,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "cfg-if 1.0.4", "chrono", @@ -8453,7 +8868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "byteorder", "chrono", @@ -8691,7 +9106,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -8765,9 +9180,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.0", - "fancy-regex", + "fancy-regex 0.11.0", "filedescriptor", "finl_unicode", "fixedbitset 0.4.2", @@ -8917,6 +9332,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str 0.9.1", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -9052,7 +9500,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dad543404f98bfc969aeb71994105c592acfc6c43323fddcd016bb208d1c65cb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-sink", @@ -9153,7 +9601,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", @@ -9182,7 +9630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", - "prost", + "prost 0.14.3", "tonic", ] @@ -9192,8 +9640,8 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" dependencies = [ - "prost", - "prost-types", + "prost 0.14.3", + "prost-types 0.14.3", "tonic", ] @@ -9482,6 +9930,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -9502,9 +9959,15 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ "itertools", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -9517,6 +9980,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" @@ -9539,6 +10008,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" @@ -10530,7 +11015,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -10567,7 +11052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 3ac7ee4cce..f7bbad294b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,10 @@ members = [ "crates/buzz-pair-relay", "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", + "crates/buzz-voice", "examples/countdown-bot", ] -exclude = ["desktop/src-tauri"] +exclude = ["desktop/src-tauri", "crates/sherpa-onnx-sys"] resolver = "2" [workspace.package] @@ -170,3 +171,4 @@ strip = true # allowlist for the auth token). Revert to crates.io once #449 lands upstream. [patch.crates-io] aws-creds = { git = "https://github.com/tlongwell-block/rust-s3", rev = "c9fce3620dd434c1f810101d672cf384268dbb0f" } +sherpa-onnx-sys = { path = "crates/sherpa-onnx-sys" } diff --git a/Dockerfile b/Dockerfile index d883ac6b01..6699273784 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update \ # locations. The normal runtime strips it below; runtime-debug retains it. ENV CARGO_PROFILE_RELEASE_DEBUG=line-tables-only COPY --from=planner /build/recipe.json recipe.json +# The sherpa sys patch keeps its published 1.13.4 version so Cargo can replace +# the registry crate. It is excluded from the workspace recipe because +# cargo-chef masks workspace package versions to 0.0.1. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys # Cook the full workspace recipe — relay deps include workspace siblings, so # scoping to -p buzz-relay misses transitive deps and re-builds them later. RUN cargo chef cook --release --recipe-path recipe.json diff --git a/Dockerfile.push-gateway b/Dockerfile.push-gateway index 202262d222..98204ff0d4 100644 --- a/Dockerfile.push-gateway +++ b/Dockerfile.push-gateway @@ -15,6 +15,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json +# Keep the patched sherpa sys crate at its published version; cargo-chef masks +# workspace package versions, so the patch remains outside the workspace recipe. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys RUN cargo chef cook --release --recipe-path recipe.json COPY . . RUN cargo build --release --locked -p buzz-push-gateway --bin buzz-push-gateway \ diff --git a/Justfile b/Justfile index bcef8983bc..e5601607ba 100644 --- a/Justfile +++ b/Justfile @@ -276,6 +276,7 @@ test-unit: #!/usr/bin/env bash if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + cargo nextest run -p buzz-voice --lib # buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra). # They guard the embedded-migrator invariant (exactly the consolidated # 0001; cutover/backfill stays an operator script, not startup state) diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml new file mode 100644 index 0000000000..6576fba57b --- /dev/null +++ b/crates/buzz-voice/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "buzz-voice" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable local voice primitives for Buzz" + +[features] +default = ["static"] +static = ["sherpa-onnx/static"] +shared = ["sherpa-onnx/shared"] + +[dependencies] +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] } +ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] } +rand = { workspace = true } +sentencepiece-model = "0.1" +serde = { workspace = true } +serde_json = { workspace = true } +sherpa-onnx = { version = "1.12", default-features = false } +tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] } diff --git a/crates/buzz-voice/examples/pocket_april_onset.rs b/crates/buzz-voice/examples/pocket_april_onset.rs new file mode 100644 index 0000000000..b2f4a02ff6 --- /dev/null +++ b/crates/buzz-voice/examples/pocket_april_onset.rs @@ -0,0 +1,75 @@ +//! Generate one raw, untrimmed April Pocket TTS onset-stress clip. +//! +//! Each sentence is a separate synthesis call, making every sentence start an +//! explicit model-generation boundary. Outputs are concatenated without fades, +//! trimming, normalization, or gain; only 250 ms of digital silence is inserted +//! between calls so boundaries are easy to identify while listening. +//! +//! Usage: +//! cargo run --release -p buzz-voice --example pocket_april_onset -- \ +//! + +use std::path::PathBuf; + +use buzz_voice::pocket::{ + load_text_to_speech, load_voice_style, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT, +}; + +const PASSAGE: &[&str] = &[ + "Yep.", + "Hello there.", + "What happened?", + "Try this now.", + "Please check again.", + "Start with the smallest change.", + "I'm listening.", + "I've got it.", + "Sounds good to me.", +]; + +fn main() -> Result<(), String> { + let mut args = std::env::args().skip(1); + let model_dir = PathBuf::from(args.next().ok_or("missing model directory argument")?); + let output = PathBuf::from(args.next().ok_or("missing output WAV argument")?); + if args.next().is_some() { + return Err("unexpected extra argument".to_string()); + } + + let engine = load_text_to_speech( + model_dir + .to_str() + .ok_or_else(|| format!("model path is not UTF-8: {}", model_dir.display()))?, + )?; + let voice_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}")); + let voice = load_voice_style(&voice_path)?; + let boundary_silence = vec![0.0_f32; SAMPLE_RATE as usize / 4]; + let mut combined = Vec::new(); + + for (index, text) in PASSAGE.iter().enumerate() { + eprintln!("boundary {}: {text}", index + 1); + let samples = engine.synth_chunk(text, "en", &voice, 1)?; + eprintln!( + " raw samples: {} ({:.3} s)", + samples.len(), + samples.len() as f32 / SAMPLE_RATE as f32 + ); + combined.extend_from_slice(&samples); + if index + 1 != PASSAGE.len() { + combined.extend_from_slice(&boundary_silence); + } + } + + let output_str = output + .to_str() + .ok_or_else(|| format!("output path is not UTF-8: {}", output.display()))?; + if !sherpa_onnx::write(output_str, &combined, SAMPLE_RATE as i32) { + return Err(format!("could not write {}", output.display())); + } + eprintln!( + "wrote {} raw samples ({:.3} s) to {}", + combined.len(), + combined.len() as f32 / SAMPLE_RATE as f32, + output.display() + ); + Ok(()) +} diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs new file mode 100644 index 0000000000..e2ce3f255b --- /dev/null +++ b/crates/buzz-voice/src/lib.rs @@ -0,0 +1,3 @@ +//! Reusable local voice primitives for Buzz. + +pub mod pocket; diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs new file mode 100644 index 0000000000..fa4a585e38 --- /dev/null +++ b/crates/buzz-voice/src/pocket.rs @@ -0,0 +1,366 @@ +//! Pocket TTS engine for Kyutai's `english_2026-04` bundle. +//! +//! Pocket TTS is a compact zero-shot voice-cloning model from Kyutai. Buzz +//! uses the full-precision April ONNX export and implements its SentencePiece, +//! learned-BOS, and recurrent-state frontend on top of the ONNX Runtime already +//! linked by sherpa-onnx. +//! +//! ## Attribution +//! +//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. +//! arXiv:2509.06926. Original repository: . +//! Licensed CC-BY-4.0. +//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. +//! - **ONNX export**: KevinAHM — +//! . CC-BY-4.0. +//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary +//! (f, conversation)" preset from the Kyutai TTS demo +//! (), which maps to `vctk/p333_023_enhanced.wav` +//! in . CC-BY-4.0, base recording +//! from the VCTK corpus, enhanced by ai-coustics. +//! +//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` +//! sidecar written by `huddle::models` during install for the canonical +//! CC-BY-4.0 §3(a)(1) attribution block. +//! +//! ## Engine-module contract (see `huddle::tts`) +//! +//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this +//! contract is what lets the TTS pipeline stay engine-agnostic: +//! +//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. +//! - `DEFAULT_VOICE: &str` — default voice name (without extension). +//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. +//! - `load_text_to_speech(model_dir)` → `Result` +//! - `load_voice_style(path)` → `Result` +//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` +//! → `Result, String>` +//! +//! `lang` and `steps` are accepted for compatibility with the shared engine +//! contract but are unused: this bundle is English-only and uses one flow step. + +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use sherpa_onnx::Wave; + +#[path = "pocket_april.rs"] +mod pocket_april; +use pocket_april::AprilPocketTts; + +// ── Engine-module contract: public consts ───────────────────────────────────── + +/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, +/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. +pub const SAMPLE_RATE: u32 = 24_000; + +/// Name (without extension) of the bundled reference voice. The model directory +/// is expected to contain `.` after install. +pub const DEFAULT_VOICE: &str = "reference_sample"; + +/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the +/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, +/// not a precomputed embedding, so the extension change is honest. +pub const VOICE_FILE_EXT: &str = "wav"; + +// ── Voice style ─────────────────────────────────────────────────────────────── + +/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. +/// +/// Pocket TTS takes a reference waveform per generation call (not a +/// precomputed style embedding), so we keep the samples in memory and clone +/// the small `Vec` into each `GenerationConfig` rather than re-reading the +/// WAV from disk on every sentence. +#[derive(Debug, Clone)] +pub struct VoiceStyle { + samples: Vec, + sample_rate: i32, +} + +/// Load a reference voice WAV from disk. +/// +/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS +/// resamples internally using `reference_sample_rate`. The bundled +/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. +pub fn load_voice_style(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; + let wave = Wave::read(path_str) + .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; + let samples = wave.samples().to_vec(); + if samples.is_empty() { + return Err(format!("voice WAV is empty: {}", path.display())); + } + Ok(VoiceStyle { + samples, + sample_rate: wave.sample_rate(), + }) +} + +// ── Engine ──────────────────────────────────────────────────────────────────── + +/// Pocket TTS engine handle, owned by the TTS worker for a huddle session. +pub struct PocketTts { + inner: Mutex, +} + +/// Build the April Pocket TTS engine from the directory installed by +/// `huddle::models`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let dir = PathBuf::from(model_dir); + Ok(PocketTts { + inner: Mutex::new(AprilPocketTts::load(&dir)?), + }) +} + +// ── Prompt preparation ──────────────────────────────────────────────────────── + +/// Result of [`prepare_pocket_prompt`]. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PreparedPrompt { + /// Capitalized, whitespace-normalized, punctuation-terminated text. + pub text: String, + /// April's upstream heuristic is 3+2 frames for ≤4 words and 1+2 otherwise. + pub frames_after_eos: usize, +} + +/// Mirror the April bundle's upstream text preparation: +/// +/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but +/// cheap to re-check after sentence splitting). +/// 2. Capitalize the first letter. +/// 3. Append `.` when the last character is alphanumeric. +/// 4. Use the model's word-count-based post-EOS frame heuristic. +/// +/// The bundle disables short-input space padding; prompts must stay unpadded +/// so phrase starts match the model's expected input distribution. +/// +/// Returns `None` only if the input is empty after trimming — caller should +/// skip synthesis in that case. +pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + // Collapse stray double-spaces / embedded newlines that may slip past + // `preprocess_for_tts` when sentences are spliced back together. + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + let is_ws = ch.is_whitespace(); + if is_ws { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. + // The upstream Python only checks `isalnum` → period, but for our agent + // text we already may end in `!` `?` `.` etc. — treat any of those as OK. + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if last.is_alphanumeric() { + cleaned.push('.'); + } + + let word_count = cleaned.split_whitespace().count(); + + Some(PreparedPrompt { + text: cleaned, + frames_after_eos: if word_count <= 4 { 5 } else { 3 }, + }) +} + +impl PocketTts { + /// Split text at word boundaries using the April bundle's exact tokenizer + /// limit. Callers should treat every returned item as an independent + /// playback chunk so cancellation and boundary processing remain visible. + pub fn split_text_into_chunks(&self, text: &str) -> Result, String> { + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(Vec::new()), + }; + + self.inner + .lock() + .map_err(|_| "Pocket TTS engine lock poisoned".to_string())? + .split_prompt(&prepared) + } + + /// Synthesise `text` with the given reference voice. + /// + /// `_lang` and `_steps` are accepted for API compatibility with the + /// previous Kokoro engine. Pocket TTS infers language from the input text + /// directly and is a one-step consistency model. Returns an empty buffer + /// for whitespace-only input. + pub fn synth_chunk( + &self, + text: &str, + lang: &str, + style: &VoiceStyle, + steps: usize, + ) -> Result, String> { + self.synth_chunk_with_callback(text, lang, style, steps, None:: bool>) + } + + /// Synthesise `text`, allowing the caller to stop generation early. + /// + /// The callback receives the samples generated so far and a progress + /// value in `[0, 1]`. Return `true` to continue or `false` to cancel. + /// Clients that do not need cancellation should use [`Self::synth_chunk`]. + pub fn synth_chunk_with_callback( + &self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + mut callback: Option, + ) -> Result, String> + where + F: FnMut(&[f32], f32) -> bool + 'static, + { + // Mirror the April bundle's prompt normalization and EOS policy. + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(Vec::new()), + }; + + self.inner + .lock() + .map_err(|_| "Pocket TTS engine lock poisoned".to_string())? + .synth_chunk( + &prepared, + style, + callback + .as_mut() + .map(|callback| callback as &mut dyn FnMut(&[f32], f32) -> bool), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── prepare_pocket_prompt ──────────────────────────────────────────────── + + #[test] + fn prepare_prompt_returns_none_for_empty_input() { + assert!(prepare_pocket_prompt("").is_none()); + assert!(prepare_pocket_prompt(" ").is_none()); + assert!(prepare_pocket_prompt("\n\t ").is_none()); + } + + #[test] + fn prepare_prompt_capitalizes_one_word_without_january_padding() { + let out = prepare_pocket_prompt("yep").expect("non-empty"); + assert_eq!(out.text, "Yep."); + assert_eq!(out.frames_after_eos, 5); + } + + #[test] + fn prepare_prompt_preserves_existing_punctuation() { + let out = prepare_pocket_prompt("yes!").expect("non-empty"); + assert_eq!(out.text, "Yes!"); + let out = prepare_pocket_prompt("really?").expect("non-empty"); + assert_eq!(out.text, "Really?"); + } + + #[test] + fn prepare_prompt_threshold_is_inclusive_at_four_words() { + // Only the post-EOS heuristic changes at the four-word boundary. + let four = prepare_pocket_prompt("one two three four").expect("non-empty"); + assert_eq!(four.text, "One two three four."); + assert_eq!(four.frames_after_eos, 5); + + let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); + assert_eq!(five.text, "One two three four five."); + assert_eq!(five.frames_after_eos, 3); + } + + #[test] + fn prepare_prompt_does_not_pad_long_text() { + let long = "This is a longer sentence that the model should handle just fine."; + let out = prepare_pocket_prompt(long).expect("non-empty"); + assert!(!out.text.starts_with(' ')); + assert_eq!(out.frames_after_eos, 3); + assert!(out.text.ends_with('.')); + } + + #[test] + fn prepare_prompt_collapses_whitespace() { + let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); + assert_eq!(out.text, "Hello world friend."); + } + + #[test] + fn prepare_prompt_does_not_double_capitalize_already_uppercase() { + let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); + assert_eq!(out.text, "HELLO there."); + } + + #[test] + fn prepare_prompt_handles_non_ascii_first_letter() { + // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce + // mojibake. + let out = prepare_pocket_prompt("дa").expect("non-empty"); + assert!(out.text.contains("Дa.")); + } + + #[test] + fn prepare_prompt_does_not_add_an_onset_prefix() { + let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); + assert_eq!(out.text, "I'm happy."); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn callback_can_cancel_before_model_inference() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let engine = load_text_to_speech(&dir).expect("load April bundle"); + let voice = + load_voice_style(&Path::new(&dir).join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"))) + .expect("load reference voice"); + let calls = Arc::new(AtomicUsize::new(0)); + let callback_calls = Arc::clone(&calls); + + let samples = engine + .synth_chunk_with_callback( + "Cancel immediately.", + "en", + &voice, + 1, + Some(move |_: &[f32], progress| { + callback_calls.fetch_add(1, Ordering::Relaxed); + assert_eq!(progress, 0.0); + false + }), + ) + .expect("cancel synthesis"); + + assert!(samples.is_empty()); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/buzz-voice/src/pocket_april.rs b/crates/buzz-voice/src/pocket_april.rs new file mode 100644 index 0000000000..3d32035e98 --- /dev/null +++ b/crates/buzz-voice/src/pocket_april.rs @@ -0,0 +1,895 @@ +//! Native ONNX loader for Pocket TTS `english_2026-04`. +//! +//! sherpa-onnx 1.13.4 remains Buzz's shared speech runtime, but its Pocket +//! frontend only understands the January vocab/token-score bundle. The April +//! export uses SentencePiece, prepends a learned BOS voice embedding, and +//! describes recurrent state tensors in `bundle.json`. This module implements +//! only that missing frontend/state loop and reuses the ONNX Runtime already +//! linked by sherpa-onnx. + +use std::borrow::Cow; +use std::f32::consts::TAU; +use std::fs; +use std::path::{Path, PathBuf}; + +use ort::session::{Session, SessionInputValue}; +use ort::value::{DynValue, Tensor}; +use rand::{Rng, RngExt}; +use sentencepiece_model::SentencePieceModel; +use serde::Deserialize; +use sherpa_onnx::LinearResampler; +use tokenizers::models::unigram::Unigram; +use tokenizers::pre_tokenizers::metaspace::{Metaspace, PrependScheme}; +use tokenizers::Tokenizer; + +use super::{PreparedPrompt, VoiceStyle}; + +const FILE_BUNDLE: &str = "bundle.json"; +const FILE_MIMI_ENCODER: &str = "mimi_encoder.onnx"; +const FILE_TEXT_CONDITIONER: &str = "text_conditioner.onnx"; +const FILE_FLOW_MAIN_FP32: &str = "flow_lm_main.onnx"; +const FILE_FLOW_FP32: &str = "flow_lm_flow.onnx"; +const FILE_MIMI_DECODER_FP32: &str = "mimi_decoder.onnx"; + +const MODEL_LANGUAGE: &str = "english_2026-04"; +const DEFAULT_TEMPERATURE: f32 = 0.7; +const EOS_LOGIT_THRESHOLD: f32 = -4.0; +const DECODER_CHUNK_FRAMES: usize = 12; +const GENERATION_PROGRESS_SHARE: f32 = 0.9; +const ONNX_THREADS: usize = 4; +const TOKENS_PER_SECOND_ESTIMATE: f32 = 3.0; +const GENERATION_SECONDS_PADDING: f32 = 2.0; + +type ProgressCallback<'a> = Option<&'a mut dyn FnMut(&[f32], f32) -> bool>; + +#[derive(Debug, Deserialize)] +struct Bundle { + schema_version: u32, + language: String, + sample_rate: usize, + frame_rate: f32, + samples_per_frame: usize, + latent_dim: usize, + conditioning_dim: usize, + insert_bos_before_voice: bool, + pad_with_spaces_for_short_inputs: bool, + remove_semicolons: bool, + model_recommended_frames_after_eos: Option, + max_token_per_chunk: usize, + tokenizer_file: String, + bos_before_voice_file: String, + flow_lm_state_manifest: Vec, + mimi_state_manifest: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct StateSpec { + input_name: String, + output_name: String, + dtype: StateDtype, + shape: Vec, + fill: StateFill, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum StateDtype { + #[serde(rename = "float32")] + Float32, + #[serde(rename = "int64")] + Int64, + Bool, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum StateFill { + Empty, + Nan, + Ones, + Zeros, +} + +struct StateValue { + spec: StateSpec, + value: DynValue, +} + +struct CachedVoice { + samples_ptr: usize, + samples_len: usize, + sample_rate: i32, + embeddings: Vec, +} + +pub(crate) struct AprilPocketTts { + bundle: Bundle, + tokenizer: Tokenizer, + bos_embedding: Vec, + mimi_encoder: Session, + text_conditioner: Session, + flow_main: Session, + flow: Session, + mimi_decoder: Session, + cached_voice: Option, +} + +impl AprilPocketTts { + pub(crate) fn load(dir: &Path) -> Result { + let bundle_path = dir.join(FILE_BUNDLE); + let bundle: Bundle = serde_json::from_slice( + &fs::read(&bundle_path) + .map_err(|err| format!("read {}: {err}", bundle_path.display()))?, + ) + .map_err(|err| format!("parse {}: {err}", bundle_path.display()))?; + + if bundle.schema_version != 2 { + return Err(format!( + "unsupported Pocket TTS bundle schema {} in {}", + bundle.schema_version, + bundle_path.display() + )); + } + if bundle.language != MODEL_LANGUAGE { + return Err(format!( + "expected Pocket TTS language {MODEL_LANGUAGE}, got {}", + bundle.language + )); + } + if bundle.sample_rate != 24_000 + || bundle.frame_rate != 12.5 + || bundle.samples_per_frame != 1_920 + || bundle.latent_dim != 32 + || bundle.conditioning_dim != 1024 + { + return Err(format!( + "unexpected Pocket TTS dimensions: sample_rate={}, frame_rate={}, samples_per_frame={}, latent_dim={}, conditioning_dim={}", + bundle.sample_rate, + bundle.frame_rate, + bundle.samples_per_frame, + bundle.latent_dim, + bundle.conditioning_dim + )); + } + if !bundle.insert_bos_before_voice { + return Err("April Pocket TTS bundle must insert BOS before voice".to_string()); + } + if bundle.pad_with_spaces_for_short_inputs + || bundle.remove_semicolons + || bundle.model_recommended_frames_after_eos.is_some() + || bundle.max_token_per_chunk != 50 + { + return Err("unsupported April Pocket TTS prompt-policy metadata".to_string()); + } + + let tokenizer_path = dir.join(&bundle.tokenizer_file); + let tokenizer = load_tokenizer(&tokenizer_path)?; + let bos_path = dir.join(&bundle.bos_before_voice_file); + let bos_embedding = read_npy_f32(&bos_path)?; + if bos_embedding.len() != bundle.conditioning_dim { + return Err(format!( + "{} has {} values; expected {}", + bos_path.display(), + bos_embedding.len(), + bundle.conditioning_dim + )); + } + + Ok(Self { + mimi_encoder: load_session(dir.join(FILE_MIMI_ENCODER))?, + text_conditioner: load_session(dir.join(FILE_TEXT_CONDITIONER))?, + flow_main: load_session(dir.join(FILE_FLOW_MAIN_FP32))?, + flow: load_session(dir.join(FILE_FLOW_FP32))?, + mimi_decoder: load_session(dir.join(FILE_MIMI_DECODER_FP32))?, + bundle, + tokenizer, + bos_embedding, + cached_voice: None, + }) + } + + pub(crate) fn split_prompt(&self, prepared: &PreparedPrompt) -> Result, String> { + if self.token_count(&prepared.text)? <= self.bundle.max_token_per_chunk { + return Ok(vec![prepared.text.clone()]); + } + + let mut chunks = Vec::new(); + let mut current = String::new(); + for word in prepared.text.split_whitespace() { + let candidate = if current.is_empty() { + word.to_string() + } else { + format!("{current} {word}") + }; + if self.prepared_token_count(&candidate)? <= self.bundle.max_token_per_chunk { + current = candidate; + continue; + } + if !current.is_empty() { + chunks.push(std::mem::take(&mut current)); + } + + if self.prepared_token_count(word)? <= self.bundle.max_token_per_chunk { + current = word.to_string(); + continue; + } + + let mut fragment = String::new(); + for ch in word.chars() { + let candidate = format!("{fragment}{ch}"); + if !fragment.is_empty() + && self.prepared_token_count(&candidate)? > self.bundle.max_token_per_chunk + { + chunks.push(std::mem::take(&mut fragment)); + } + fragment.push(ch); + } + current = fragment; + } + if !current.is_empty() { + chunks.push(current); + } + + chunks + .into_iter() + .map(|text| { + let chunk = super::prepare_pocket_prompt(&text) + .ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?; + let token_count = self.token_count(&chunk.text)?; + if token_count > self.bundle.max_token_per_chunk { + return Err(format!( + "Pocket TTS prompt chunk has {token_count} tokens; maximum is {}", + self.bundle.max_token_per_chunk + )); + } + Ok(chunk.text) + }) + .collect() + } + + pub(crate) fn synth_chunk( + &mut self, + prepared: &PreparedPrompt, + style: &VoiceStyle, + mut callback: ProgressCallback<'_>, + ) -> Result, String> { + if !should_continue(&mut callback, &[], 0.0) { + return Ok(Vec::new()); + } + + let voice_embeddings = self.voice_embeddings(style)?; + let mut flow_state = self.condition_voice(&voice_embeddings)?; + let token_ids = self + .tokenizer + .encode(prepared.text.as_str(), false) + .map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))? + .get_ids() + .iter() + .copied() + .map(i64::from) + .collect::>(); + if token_ids.is_empty() { + return Ok(Vec::new()); + } + if token_ids.len() > self.bundle.max_token_per_chunk { + return Err(format!( + "Pocket TTS prompt has {} tokens; split_text_into_chunks maximum is {}", + token_ids.len(), + self.bundle.max_token_per_chunk + )); + } + + let token_count = token_ids.len(); + let text_embeddings = self.text_embeddings(token_ids)?; + self.run_flow_main_prefix(&text_embeddings, &mut flow_state)?; + let max_frames = estimate_max_frames(token_count, self.bundle.frame_rate); + let Some(latents) = self.generate_latents( + max_frames, + prepared.frames_after_eos, + &mut flow_state, + &mut callback, + )? + else { + return Ok(Vec::new()); + }; + Ok(self + .decode_latents(&latents, &mut callback)? + .unwrap_or_default()) + } + + fn prepared_token_count(&self, text: &str) -> Result { + let prepared = super::prepare_pocket_prompt(text) + .ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?; + self.token_count(&prepared.text) + } + + fn token_count(&self, text: &str) -> Result { + Ok(self + .tokenizer + .encode(text, false) + .map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))? + .get_ids() + .len()) + } + + fn voice_embeddings(&mut self, style: &VoiceStyle) -> Result, String> { + let key = ( + style.samples.as_ptr() as usize, + style.samples.len(), + style.sample_rate, + ); + if let Some(cached) = &self.cached_voice { + if (cached.samples_ptr, cached.samples_len, cached.sample_rate) == key { + return Ok(cached.embeddings.clone()); + } + } + + let samples = if style.sample_rate == self.bundle.sample_rate as i32 { + style.samples.clone() + } else { + LinearResampler::create(style.sample_rate, self.bundle.sample_rate as i32) + .ok_or_else(|| { + format!( + "create Pocket TTS resampler {}Hz -> {}Hz", + style.sample_rate, self.bundle.sample_rate + ) + })? + .resample(&style.samples, true) + }; + let audio = Tensor::from_array(( + vec![1_i64, 1, samples.len() as i64], + samples.into_boxed_slice(), + )) + .map_err(ort_error("create voice audio tensor"))?; + let outputs = self + .mimi_encoder + .run(ort::inputs!["audio" => audio]) + .map_err(ort_error("run Mimi encoder"))?; + let (_, encoded) = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Mimi encoder output"))?; + if !encoded.len().is_multiple_of(self.bundle.conditioning_dim) { + return Err(format!( + "Mimi encoder returned {} values, not divisible by {}", + encoded.len(), + self.bundle.conditioning_dim + )); + } + let mut embeddings = + Vec::with_capacity(self.bos_embedding.len().saturating_add(encoded.len())); + embeddings.extend_from_slice(&self.bos_embedding); + embeddings.extend_from_slice(encoded); + self.cached_voice = Some(CachedVoice { + samples_ptr: key.0, + samples_len: key.1, + sample_rate: key.2, + embeddings: embeddings.clone(), + }); + Ok(embeddings) + } + + fn condition_voice(&mut self, embeddings: &[f32]) -> Result, String> { + let frames = embeddings.len() / self.bundle.conditioning_dim; + let sequence = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.latent_dim as i64], + ) + .map_err(ort_error("create empty voice sequence"))?; + let text_embeddings = Tensor::from_array(( + vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64], + embeddings.to_vec().into_boxed_slice(), + )) + .map_err(ort_error("create voice embedding tensor"))?; + let mut state = initialize_state(&self.bundle.flow_lm_state_manifest)?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, &state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("condition Pocket TTS voice"))?; + replace_state_from_outputs(&mut state, &mut outputs)?; + Ok(state) + } + + fn text_embeddings(&mut self, token_ids: Vec) -> Result, String> { + let tokens = Tensor::from_array(( + vec![1_i64, token_ids.len() as i64], + token_ids.into_boxed_slice(), + )) + .map_err(ort_error("create token tensor"))?; + let outputs = self + .text_conditioner + .run(ort::inputs!["token_ids" => tokens]) + .map_err(ort_error("run text conditioner"))?; + let (_, embeddings) = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract text embeddings"))?; + Ok(embeddings.to_vec()) + } + + fn run_flow_main_prefix( + &mut self, + text_embeddings: &[f32], + state: &mut [StateValue], + ) -> Result<(), String> { + if !text_embeddings + .len() + .is_multiple_of(self.bundle.conditioning_dim) + { + return Err(format!( + "text conditioner returned {} values, not divisible by {}", + text_embeddings.len(), + self.bundle.conditioning_dim + )); + } + let frames = text_embeddings.len() / self.bundle.conditioning_dim; + let sequence = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.latent_dim as i64], + ) + .map_err(ort_error("create empty text sequence"))?; + let text_embeddings = Tensor::from_array(( + vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64], + text_embeddings.to_vec().into_boxed_slice(), + )) + .map_err(ort_error("create text embedding tensor"))?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("prime Pocket TTS text state"))?; + replace_state_from_outputs(state, &mut outputs) + } + + fn generate_latents( + &mut self, + max_frames: usize, + frames_after_eos: usize, + state: &mut [StateValue], + callback: &mut ProgressCallback<'_>, + ) -> Result>, String> { + let mut current = vec![f32::NAN; self.bundle.latent_dim]; + let mut latents = Vec::with_capacity(max_frames * self.bundle.latent_dim); + let mut eos_step = None; + let mut rng = rand::rng(); + + for step in 0..max_frames { + let progress = GENERATION_PROGRESS_SHARE * (step as f32 / max_frames.max(1) as f32); + if !should_continue(callback, &[], progress) { + return Ok(None); + } + + let sequence = Tensor::from_array(( + vec![1_i64, 1, self.bundle.latent_dim as i64], + current.clone().into_boxed_slice(), + )) + .map_err(ort_error("create latent input"))?; + let text_embeddings = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.conditioning_dim as i64], + ) + .map_err(ort_error("create empty text input"))?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("run Pocket TTS Flow LM"))?; + let conditioning = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Flow LM conditioning"))? + .1 + .to_vec(); + let eos_logit = outputs[1] + .try_extract_tensor::() + .map_err(ort_error("extract Flow LM EOS logit"))? + .1 + .first() + .copied() + .ok_or_else(|| "Flow LM returned empty EOS logit".to_string())?; + replace_state_from_outputs(state, &mut outputs)?; + + if eos_logit > EOS_LOGIT_THRESHOLD && eos_step.is_none() { + eos_step = Some(step); + } + if eos_step.is_some_and(|eos| step >= eos + frames_after_eos) { + break; + } + + let mut noise = + normal_noise(&mut rng, self.bundle.latent_dim, DEFAULT_TEMPERATURE.sqrt()); + let conditioning = Tensor::from_array(( + vec![1_i64, self.bundle.conditioning_dim as i64], + conditioning.into_boxed_slice(), + )) + .map_err(ort_error("create flow conditioning"))?; + let s = Tensor::from_array((vec![1_i64, 1], vec![0.0_f32].into_boxed_slice())) + .map_err(ort_error("create flow start tensor"))?; + let t = Tensor::from_array((vec![1_i64, 1], vec![1.0_f32].into_boxed_slice())) + .map_err(ort_error("create flow end tensor"))?; + let x = Tensor::from_array(( + vec![1_i64, self.bundle.latent_dim as i64], + noise.clone().into_boxed_slice(), + )) + .map_err(ort_error("create flow noise tensor"))?; + let outputs = self + .flow + .run(ort::inputs![ + "c" => conditioning, + "s" => s, + "t" => t, + "x" => x, + ]) + .map_err(ort_error("run Pocket TTS flow"))?; + let flow = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Pocket TTS flow"))? + .1; + if flow.len() != noise.len() { + return Err(format!( + "flow returned {} values; expected {}", + flow.len(), + noise.len() + )); + } + for (sample, delta) in noise.iter_mut().zip(flow) { + *sample += *delta; + } + current.clone_from(&noise); + latents.extend_from_slice(&noise); + } + Ok(Some(latents)) + } + + fn decode_latents( + &mut self, + latents: &[f32], + callback: &mut ProgressCallback<'_>, + ) -> Result>, String> { + if latents.is_empty() { + return Ok(Some(Vec::new())); + } + if !latents.len().is_multiple_of(self.bundle.latent_dim) { + return Err(format!( + "latent buffer has {} values, not divisible by {}", + latents.len(), + self.bundle.latent_dim + )); + } + let frame_count = latents.len() / self.bundle.latent_dim; + let mut state = initialize_state(&self.bundle.mimi_state_manifest)?; + let mut audio = Vec::new(); + + for start in (0..frame_count).step_by(DECODER_CHUNK_FRAMES) { + let end = (start + DECODER_CHUNK_FRAMES).min(frame_count); + let values = + latents[start * self.bundle.latent_dim..end * self.bundle.latent_dim].to_vec(); + let latent = Tensor::from_array(( + vec![1_i64, (end - start) as i64, self.bundle.latent_dim as i64], + values.into_boxed_slice(), + )) + .map_err(ort_error("create Mimi latent tensor"))?; + let mut inputs = vec![(Cow::Borrowed("latent"), SessionInputValue::from(latent))]; + append_state_inputs(&mut inputs, &state); + let mut outputs = self + .mimi_decoder + .run(inputs) + .map_err(ort_error("run Mimi decoder"))?; + let samples = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Mimi audio"))? + .1; + audio.extend_from_slice(samples); + replace_state_from_outputs(&mut state, &mut outputs)?; + + let progress = GENERATION_PROGRESS_SHARE + + (1.0 - GENERATION_PROGRESS_SHARE) * (end as f32 / frame_count as f32); + if !should_continue(callback, &audio, progress) { + return Ok(None); + } + } + Ok(Some(audio)) + } +} + +fn should_continue(callback: &mut ProgressCallback<'_>, samples: &[f32], progress: f32) -> bool { + callback + .as_mut() + .is_none_or(|callback| callback(samples, progress.clamp(0.0, 1.0))) +} + +fn load_session(path: PathBuf) -> Result { + if !path.is_file() { + return Err(format!("missing Pocket TTS file: {}", path.display())); + } + Session::builder() + .map_err(ort_error("create ONNX session builder"))? + .with_intra_threads(ONNX_THREADS) + .map_err(|err| format!("configure ONNX intra-op threads: {err}"))? + .with_inter_threads(1) + .map_err(|err| format!("configure ONNX inter-op threads: {err}"))? + .commit_from_file(&path) + .map_err(|err| format!("load {}: {err}", path.display())) +} + +fn load_tokenizer(path: &Path) -> Result { + let sentencepiece = SentencePieceModel::from_file(path) + .map_err(|err| format!("load {}: {err}", path.display()))?; + let trainer = sentencepiece + .trainer() + .ok_or_else(|| format!("{} has no SentencePiece trainer metadata", path.display()))?; + let normalizer = sentencepiece.normalizer().ok_or_else(|| { + format!( + "{} has no SentencePiece normalizer metadata", + path.display() + ) + })?; + if normalizer.name() != "identity" { + return Err(format!( + "{} uses unsupported SentencePiece normalizer {:?}", + path.display(), + normalizer.name() + )); + } + + let vocab = sentencepiece + .pieces() + .iter() + .map(|piece| (piece.piece().to_owned(), f64::from(piece.score()))) + .collect(); + let mut tokenizer = Tokenizer::new( + Unigram::from( + vocab, + Some(trainer.unk_id() as usize), + trainer.byte_fallback(), + ) + .map_err(|err| format!("construct tokenizer from {}: {err}", path.display()))?, + ); + // SentencePiece's identity normalizer still escapes spaces as U+2581 and + // prepends one marker to the input before unigram segmentation. + tokenizer.with_pre_tokenizer(Some(Metaspace::new('▁', PrependScheme::Always, false))); + Ok(tokenizer) +} + +fn initialize_state(specs: &[StateSpec]) -> Result, String> { + specs + .iter() + .cloned() + .map(|spec| { + let len = shape_len(&spec.shape)?; + let value = match spec.dtype { + StateDtype::Float32 => { + let fill = match spec.fill { + StateFill::Nan => f32::NAN, + StateFill::Empty | StateFill::Zeros => 0.0, + StateFill::Ones => 1.0, + }; + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty float state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create float state tensor"))? + .into_dyn() + } + } + StateDtype::Int64 => { + let fill = i64::from(matches!(spec.fill, StateFill::Ones)); + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty integer state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create integer state tensor"))? + .into_dyn() + } + } + StateDtype::Bool => { + let fill = matches!(spec.fill, StateFill::Ones); + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty bool state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create bool state tensor"))? + .into_dyn() + } + } + }; + Ok(StateValue { spec, value }) + }) + .collect() +} + +fn append_state_inputs<'a>( + inputs: &mut Vec<(Cow<'a, str>, SessionInputValue<'a>)>, + state: &'a [StateValue], +) { + for value in state { + inputs.push(( + Cow::Borrowed(value.spec.input_name.as_str()), + SessionInputValue::from(&value.value), + )); + } +} + +fn replace_state_from_outputs( + state: &mut [StateValue], + outputs: &mut ort::session::SessionOutputs<'_>, +) -> Result<(), String> { + for value in state { + value.value = outputs + .remove(&value.spec.output_name) + .ok_or_else(|| format!("missing state output {}", value.spec.output_name))?; + } + Ok(()) +} + +fn shape_len(shape: &[i64]) -> Result { + shape.iter().try_fold(1_usize, |len, &dim| { + let dim = usize::try_from(dim).map_err(|_| format!("negative state dimension {dim}"))?; + len.checked_mul(dim) + .ok_or_else(|| format!("state shape overflows usize: {shape:?}")) + }) +} + +fn estimate_max_frames(token_count: usize, frame_rate: f32) -> usize { + ((token_count as f32 / TOKENS_PER_SECOND_ESTIMATE + GENERATION_SECONDS_PADDING) * frame_rate) + .ceil() as usize +} + +fn normal_noise(rng: &mut impl Rng, len: usize, std_dev: f32) -> Vec { + let mut out = Vec::with_capacity(len); + while out.len() < len { + let u1 = rng.random::().max(f32::MIN_POSITIVE); + let u2 = rng.random::(); + let radius = (-2.0_f32 * u1.ln()).sqrt() * std_dev; + out.push(radius * (TAU * u2).cos()); + if out.len() < len { + out.push(radius * (TAU * u2).sin()); + } + } + out +} + +fn read_npy_f32(path: &Path) -> Result, String> { + let bytes = fs::read(path).map_err(|err| format!("read {}: {err}", path.display()))?; + if bytes.len() < 10 || &bytes[..6] != b"\x93NUMPY" { + return Err(format!("{} is not a NumPy array", path.display())); + } + let major = bytes[6]; + let header_len_bytes = match major { + 1 => 2, + 2 | 3 => 4, + _ => { + return Err(format!( + "unsupported NumPy version {major} in {}", + path.display() + )) + } + }; + let header_start = 8 + header_len_bytes; + if bytes.len() < header_start { + return Err(format!("truncated NumPy header in {}", path.display())); + } + let header_len = if header_len_bytes == 2 { + u16::from_le_bytes([bytes[8], bytes[9]]) as usize + } else { + u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize + }; + let data_start = header_start + .checked_add(header_len) + .ok_or_else(|| format!("NumPy header overflow in {}", path.display()))?; + if data_start > bytes.len() { + return Err(format!("truncated NumPy data in {}", path.display())); + } + let header = std::str::from_utf8(&bytes[header_start..data_start]) + .map_err(|err| format!("invalid NumPy header in {}: {err}", path.display()))?; + if !(header.contains("'descr': ' impl FnOnce(ort::Error) -> String { + move |err| format!("{context}: {err}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shape_len_supports_empty_state_dimensions() { + assert_eq!(shape_len(&[1, 128, 0]).expect("shape"), 0); + assert_eq!(shape_len(&[2, 1, 8, 1000, 64]).expect("shape"), 1_024_000); + } + + #[test] + fn normal_noise_has_requested_length() { + let mut rng = rand::rng(); + assert_eq!(normal_noise(&mut rng, 1, 1.0).len(), 1); + assert_eq!(normal_noise(&mut rng, 32, 1.0).len(), 32); + } + + #[test] + fn generation_frame_estimate_scales_with_token_count() { + assert_eq!(estimate_max_frames(3, 12.5), 38); + assert_eq!(estimate_max_frames(300, 12.5), 1_275); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn tokenizer_matches_sentencepiece_reference_including_unknown_words() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let tokenizer = + load_tokenizer(&Path::new(&dir).join("tokenizer.model")).expect("load April tokenizer"); + let cases: &[(&str, &[u32])] = &[ + ("Yep.", &[2462, 263]), + ("Hello there.", &[2994, 310, 263]), + ( + "quizzaciously xyzzy.", + &[ + 260, 1157, 1818, 362, 1814, 323, 260, 568, 327, 1818, 327, 263, + ], + ), + ("I'm listening.", &[268, 264, 283, 260, 604, 273, 263]), + ]; + for (text, expected) in cases { + let encoding = tokenizer.encode(*text, false).expect("tokenize"); + assert_eq!(encoding.get_ids(), *expected, "{text}"); + } + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn loader_splits_oversized_prompts_at_bundle_token_limit() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let engine = AprilPocketTts::load(Path::new(&dir)).expect("load April bundle"); + let text = "This deliberately long sentence repeats ordinary English words so the exact SentencePiece token limit is exercised without relying on punctuation, and it keeps adding more material until the prompt must be divided into multiple independently safe generation chunks before the recurrent state cache can be exhausted."; + let prepared = crate::pocket::prepare_pocket_prompt(text).expect("prepare prompt"); + let chunks = engine.split_prompt(&prepared).expect("split prompt"); + + assert!(chunks.len() > 1); + assert!(chunks.iter().all(|chunk| { + engine.token_count(chunk).expect("tokenize chunk") <= engine.bundle.max_token_per_chunk + })); + } +} diff --git a/crates/sherpa-onnx-sys/Cargo.toml b/crates/sherpa-onnx-sys/Cargo.toml new file mode 100644 index 0000000000..317d2c5b26 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml @@ -0,0 +1,63 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "sherpa-onnx-sys" +version = "1.13.4" +build = "build.rs" +links = "sherpa-onnx" +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Raw FFI bindings to the sherpa-onnx C API" +readme = "README.md" +keywords = [ + "ffi", + "speech", + "sherpa-onnx", + "bindings", +] +categories = ["external-ffi-bindings"] +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" + +[package.metadata.docs.rs] +default-features = false +features = [] + +[features] +default = ["static"] +shared = [] +static = [] + +[lib] +name = "sherpa_onnx_sys" +path = "src/lib.rs" + +[build-dependencies.bzip2] +version = "0.4" + +[build-dependencies.tar] +version = "0.4" + +[build-dependencies.ureq] +version = "2.12" +features = ["proxy-from-env"] diff --git a/crates/sherpa-onnx-sys/Cargo.toml.orig b/crates/sherpa-onnx-sys/Cargo.toml.orig new file mode 100644 index 0000000000..5744d49363 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +name = "sherpa-onnx-sys" +version = "1.13.4" +edition = "2021" +description = "Raw FFI bindings to the sherpa-onnx C API" +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" +readme = "README.md" +links = "sherpa-onnx" + +keywords = ["ffi", "speech", "sherpa-onnx", "bindings"] +categories = ["external-ffi-bindings"] + +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] + +[features] +default = ["static"] +static = [] +shared = [] + +[build-dependencies] +bzip2 = "0.4" +tar = "0.4" +ureq = { version = "2.12", features = ["proxy-from-env"] } + +[package.metadata.docs.rs] +default-features = false +features = [] diff --git a/crates/sherpa-onnx-sys/LICENSE b/crates/sherpa-onnx-sys/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/crates/sherpa-onnx-sys/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sherpa-onnx-sys/README.md b/crates/sherpa-onnx-sys/README.md new file mode 100644 index 0000000000..689fa55575 --- /dev/null +++ b/crates/sherpa-onnx-sys/README.md @@ -0,0 +1,671 @@ +
+ +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/k2-fsa/sherpa-onnx) + +
+ +Buzz carries the crates.io 1.13.4 FFI sources and locally patches the build +script to link caller-supplied official iOS and Android libraries through +`SHERPA_ONNX_LIB_DIR`. Desktop archive selection remains unchanged. + + ### Supported functions + +|Speech recognition| [Speech synthesis][tts-url] | [Source separation][ss-url] | +|------------------|------------------|-------------------| +| ✔️ | ✔️ | ✔️ | + +|Speaker identification| [Speaker diarization][sd-url] | Speaker verification | +|----------------------|-------------------- |------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Spoken Language identification][slid-url] | [Audio tagging][at-url] | [Voice activity detection][vad-url] | +|--------------------------------|---------------|--------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Keyword spotting][kws-url] | [Add punctuation][punct-url] | [Speech enhancement][se-url] | +|------------------|-----------------|--------------------| +| ✔️ | ✔️ | ✔️ | + + +### Supported platforms + +|Architecture| Android | iOS | Windows | macOS | linux | HarmonyOS | +|------------|---------|---------|------------|-------|-------|-----------| +| x64 | ✔️ | | ✔️ | ✔️ | ✔️ | ✔️ | +| x86 | ✔️ | | ✔️ | | | | +| arm64 | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | +| arm32 | ✔️ | | | | ✔️ | ✔️ | +| riscv64 | | | | | ✔️ | | + +### Supported programming languages + +| 1. C++ | 2. C | 3. Python | 4. JavaScript | +|--------|-------|-----------|---------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +|5. Java | 6. C# | 7. Kotlin | 8. Swift | +|--------|-------|-----------|----------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +| 9. Go | 10. Dart | 11. Rust | 12. Pascal | +|-------|----------|----------|------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + + +It also supports WebAssembly. + +### Supported NPUs + +| [1. Rockchip NPU (RKNN)][rknpu-doc] | [2. Qualcomm NPU (QNN)][qnn-doc] | [3. Ascend NPU][ascend-doc] | +|-------------------------------------|-----------------------------------|-----------------------------| +| ✔️ | ✔️ | ✔️ | + +| [4. Axera NPU][axera-npu] | +|---------------------------| +| ✔️ | + +[Join our discord](https://discord.gg/fJdxzg2VbG) + + +## Introduction + +This repository supports running the following functions **locally** + + - Speech-to-text (i.e., ASR); both streaming and non-streaming are supported + - Text-to-speech (i.e., TTS) + - Speaker diarization + - Speaker identification + - Speaker verification + - Spoken language identification + - Audio tagging + - VAD (e.g., [silero-vad][silero-vad]) + - Speech enhancement (e.g., [gtcrn][gtcrn], [DPDFNet](https://github.com/ceva-ip/DPDFNet)) + - Keyword spotting + - Source separation (e.g., [spleeter][spleeter], [UVR][UVR]) + +on the following platforms and operating systems: + + - x86, ``x86_64``, 32-bit ARM, 64-bit ARM (arm64, aarch64), RISC-V (riscv64), **RK NPU**, **Ascend NPU** + - Linux, macOS, Windows, openKylin + - Android, WearOS + - iOS + - HarmonyOS + - NodeJS + - WebAssembly + - [NVIDIA Jetson Orin NX][NVIDIA Jetson Orin NX] (Support running on both CPU and GPU) + - [NVIDIA Jetson Nano B01][NVIDIA Jetson Nano B01] (Support running on both CPU and GPU) + - [Raspberry Pi][Raspberry Pi] + - [RV1126][RV1126] + - [LicheePi4A][LicheePi4A] + - [VisionFive 2][VisionFive 2] + - [旭日X3派][旭日X3派] + - [爱芯派][爱芯派] + - [RK3588][RK3588] + - [SpacemiT-K1][SpacemiT-K1] + - [SpacemiT-K3][SpacemiT-K3] + - etc + +with the following APIs + + - C++, C, Python, Go, ``C#`` + - Java, Kotlin, JavaScript + - Swift, Rust + - Dart, Object Pascal + +### Links for Huggingface Spaces + +
+You can visit the following Huggingface spaces to try sherpa-onnx without +installing anything. All you need is a browser. + +| Description | URL | 中国镜像 | +|-------------------------------------------------------|-----------------------------------------|----------------------------------------| +| Speaker diarization | [Click me][hf-space-speaker-diarization]| [镜像][hf-space-speaker-diarization-cn]| +| Speech recognition | [Click me][hf-space-asr] | [镜像][hf-space-asr-cn] | +| Speech recognition with [Whisper][Whisper] | [Click me][hf-space-asr-whisper] | [镜像][hf-space-asr-whisper-cn] | +| Speech synthesis | [Click me][hf-space-tts] | [镜像][hf-space-tts-cn] | +| Generate subtitles | [Click me][hf-space-subtitle] | [镜像][hf-space-subtitle-cn] | +| Audio tagging | [Click me][hf-space-audio-tagging] | [镜像][hf-space-audio-tagging-cn] | +| Source separation | [Click me][hf-space-source-separation] | [镜像][hf-space-source-separation-cn] | +| Spoken language identification with [Whisper][Whisper]| [Click me][hf-space-slid-whisper] | [镜像][hf-space-slid-whisper-cn] | + +We also have spaces built using WebAssembly. They are listed below: + +| Description | Huggingface space| ModelScope space| +|------------------------------------------------------------------------------------------|------------------|-----------------| +|Voice activity detection with [silero-vad][silero-vad] | [Click me][wasm-hf-vad]|[地址][wasm-ms-vad]| +|Real-time speech recognition (Chinese + English) with Zipformer | [Click me][wasm-hf-streaming-asr-zh-en-zipformer]|[地址][wasm-hf-streaming-asr-zh-en-zipformer]| +|Real-time speech recognition (Chinese + English) with Paraformer |[Click me][wasm-hf-streaming-asr-zh-en-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-paraformer]| +|Real-time speech recognition (Chinese + English + Cantonese) with [Paraformer-large][Paraformer-large]|[Click me][wasm-hf-streaming-asr-zh-en-yue-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-yue-paraformer]| +|Real-time speech recognition (English) |[Click me][wasm-hf-streaming-asr-en-zipformer] |[地址][wasm-ms-streaming-asr-en-zipformer]| +|VAD + speech recognition (Chinese) with [Zipformer CTC](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|[Click me][wasm-hf-vad-asr-zh-zipformer-ctc-07-03]| [地址][wasm-ms-vad-asr-zh-zipformer-ctc-07-03]| +|VAD + speech recognition (Chinese + English + Korean + Japanese + Cantonese) with [SenseVoice][SenseVoice]|[Click me][wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]| [地址][wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]| +|VAD + speech recognition (English) with [Whisper][Whisper] tiny.en|[Click me][wasm-hf-vad-asr-en-whisper-tiny-en]| [地址][wasm-ms-vad-asr-en-whisper-tiny-en]| +|VAD + speech recognition (English) with [Moonshine tiny][Moonshine tiny]|[Click me][wasm-hf-vad-asr-en-moonshine-tiny-en]| [地址][wasm-ms-vad-asr-en-moonshine-tiny-en]| +|VAD + speech recognition (English) with Zipformer trained with [GigaSpeech][GigaSpeech] |[Click me][wasm-hf-vad-asr-en-zipformer-gigaspeech]| [地址][wasm-ms-vad-asr-en-zipformer-gigaspeech]| +|VAD + speech recognition (Chinese) with Zipformer trained with [WenetSpeech][WenetSpeech] |[Click me][wasm-hf-vad-asr-zh-zipformer-wenetspeech]| [地址][wasm-ms-vad-asr-zh-zipformer-wenetspeech]| +|VAD + speech recognition (Japanese) with Zipformer trained with [ReazonSpeech][ReazonSpeech]|[Click me][wasm-hf-vad-asr-ja-zipformer-reazonspeech]| [地址][wasm-ms-vad-asr-ja-zipformer-reazonspeech]| +|VAD + speech recognition (Thai) with Zipformer trained with [GigaSpeech2][GigaSpeech2] |[Click me][wasm-hf-vad-asr-th-zipformer-gigaspeech2]| [地址][wasm-ms-vad-asr-th-zipformer-gigaspeech2]| +|VAD + speech recognition (Chinese 多种方言) with a [TeleSpeech-ASR][TeleSpeech-ASR] CTC model|[Click me][wasm-hf-vad-asr-zh-telespeech]| [地址][wasm-ms-vad-asr-zh-telespeech]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-large |[Click me][wasm-hf-vad-asr-zh-en-paraformer-large]| [地址][wasm-ms-vad-asr-zh-en-paraformer-large]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-small |[Click me][wasm-hf-vad-asr-zh-en-paraformer-small]| [地址][wasm-ms-vad-asr-zh-en-paraformer-small]| +|VAD + speech recognition (多语种及多种中文方言) with [Dolphin][Dolphin]-base |[Click me][wasm-hf-vad-asr-multi-lang-dolphin-base]| [地址][wasm-ms-vad-asr-multi-lang-dolphin-base]| +|Speech synthesis (Piper, English) |[Click me][wasm-hf-tts-piper-en]| [地址][wasm-ms-tts-piper-en]| +|Speech synthesis (Piper, German) |[Click me][wasm-hf-tts-piper-de]| [地址][wasm-ms-tts-piper-de]| +|Speech synthesis (Matcha, Chinese) |[Click me][wasm-hf-tts-matcha-zh]| [地址][wasm-ms-tts-matcha-zh]| +|Speech synthesis (Matcha, English) |[Click me][wasm-hf-tts-matcha-en]| [地址][wasm-ms-tts-matcha-en]| +|Speech synthesis (Matcha, Chinese+English) |[Click me][wasm-hf-tts-matcha-zh-en]| [地址][wasm-ms-tts-matcha-zh-en]| +|Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]| +|Voice cloning with ZipVoice (Chinese+English) |[Click me][wasm-hf-voice-cloning-zipvoice]|[地址][wasm-ms-voice-cloning-zipvoice]| +|Voice cloning with Pocket TTS (English) |[Click me][wasm-hf-voice-cloning-pocket]|[地址][wasm-ms-voice-cloning-pocket]| + +
+ +### Links for pre-built Android APKs + +
+ +You can find pre-built Android APKs for this repository in the following table + +| Description | URL | 中国用户 | +|----------------------------------------|------------------------------------|-----------------------------------| +| Speaker diarization | [Address][apk-speaker-diarization] | [点此][apk-speaker-diarization-cn]| +| Streaming speech recognition | [Address][apk-streaming-asr] | [点此][apk-streaming-asr-cn] | +| Simulated-streaming speech recognition | [Address][apk-simula-streaming-asr]| [点此][apk-simula-streaming-asr-cn]| +| Text-to-speech | [Address][apk-tts] | [点此][apk-tts-cn] | +| Voice activity detection (VAD) | [Address][apk-vad] | [点此][apk-vad-cn] | +| VAD + non-streaming speech recognition | [Address][apk-vad-asr] | [点此][apk-vad-asr-cn] | +| Two-pass speech recognition | [Address][apk-2pass] | [点此][apk-2pass-cn] | +| Audio tagging | [Address][apk-at] | [点此][apk-at-cn] | +| Audio tagging (WearOS) | [Address][apk-at-wearos] | [点此][apk-at-wearos-cn] | +| Speaker identification | [Address][apk-sid] | [点此][apk-sid-cn] | +| Spoken language identification | [Address][apk-slid] | [点此][apk-slid-cn] | +| Keyword spotting | [Address][apk-kws] | [点此][apk-kws-cn] | + +
+ +### Links for pre-built Flutter APPs + +
+ +#### Real-time speech recognition + +| Description | URL | 中国用户 | +|--------------------------------|-------------------------------------|-------------------------------------| +| Streaming speech recognition | [Address][apk-flutter-streaming-asr]| [点此][apk-flutter-streaming-asr-cn]| + +#### Text-to-speech + +| Description | URL | 中国用户 | +|------------------------------------------|------------------------------------|------------------------------------| +| Android (arm64-v8a, armeabi-v7a, x86_64) | [Address][flutter-tts-android] | [点此][flutter-tts-android-cn] | +| Linux (x64) | [Address][flutter-tts-linux] | [点此][flutter-tts-linux-cn] | +| macOS (x64) | [Address][flutter-tts-macos-x64] | [点此][flutter-tts-macos-x64-cn] | +| macOS (arm64) | [Address][flutter-tts-macos-arm64] | [点此][flutter-tts-macos-arm64-cn] | +| Windows (x64) | [Address][flutter-tts-win-x64] | [点此][flutter-tts-win-x64-cn] | + +> Note: You need to build from source for iOS. + +
+ +### Links for pre-built Lazarus APPs + +
+ +#### Generating subtitles + +| Description | URL | 中国用户 | +|--------------------------------|----------------------------|----------------------------| +| Generate subtitles (生成字幕) | [Address][lazarus-subtitle]| [点此][lazarus-subtitle-cn]| + +
+ +### Links for pre-trained models + +
+ +| Description | URL | +|---------------------------------------------|---------------------------------------------------------------------------------------| +| Speech recognition (speech to text, ASR) | [Address][asr-models] | +| Text-to-speech (TTS) | [Address][tts-models] | +| VAD | [Address][vad-models] | +| Keyword spotting | [Address][kws-models] | +| Audio tagging | [Address][at-models] | +| Speaker identification (Speaker ID) | [Address][sid-models] | +| Spoken language identification (Language ID)| See multi-lingual [Whisper][Whisper] ASR models from [Speech recognition][asr-models]| +| Punctuation | [Address][punct-models] | +| Speaker segmentation | [Address][speaker-segmentation-models] | +| Speech enhancement | [Address][speech-enhancement-models] | +| Source separation | [Address][source-separation-models] | + +
+ +#### Some pre-trained ASR models (Streaming) + +
+ +Please see + + - + - + - + +for more models. The following table lists only **SOME** of them. + + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20][sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16][sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23][sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]|Chinese| Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23)| +|[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17][sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]|English|Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-en-20m-2023-02-17)| +|[sherpa-onnx-streaming-zipformer-korean-2024-06-16][sherpa-onnx-streaming-zipformer-korean-2024-06-16]|Korean| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-korean-2024-06-16-korean)| +|[sherpa-onnx-streaming-zipformer-fr-2023-04-14][sherpa-onnx-streaming-zipformer-fr-2023-04-14]|French| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#shaojieli-sherpa-onnx-streaming-zipformer-fr-2023-04-14-french)| + +
+ + +#### Some pre-trained ASR models (Non-Streaming) + +
+ +Please see + + - + - + - + - + - + +for more models. The following table lists only **SOME** of them. + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-parakeet-tdt-0-6b-v2-int8-english)| English | It is converted from | +|[Whisper tiny.en](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2)|English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html)| +|[Moonshine tiny][Moonshine tiny]|English|See [also](https://github.com/usefulsensors/moonshine)| +|[sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|Chinese| A Zipformer CTC model| +|[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17][sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]|Chinese, Cantonese, English, Korean, Japanese| 支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html)| +|[sherpa-onnx-paraformer-zh-2024-03-09][sherpa-onnx-paraformer-zh-2024-03-09]|Chinese, English| 也支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2024-03-09-chinese-english)| +|[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01][sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]|Japanese|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01-japanese)| +|[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24][sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24-russian)| +|[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24][sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]|Russian| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/nemo/russian.html#sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24)| +|[sherpa-onnx-zipformer-ru-2024-09-18][sherpa-onnx-zipformer-ru-2024-09-18]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ru-2024-09-18-russian)| +|[sherpa-onnx-zipformer-korean-2024-06-24][sherpa-onnx-zipformer-korean-2024-06-24]|Korean|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-korean-2024-06-24-korean)| +|[sherpa-onnx-zipformer-thai-2024-06-20][sherpa-onnx-zipformer-thai-2024-06-20]|Thai| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-thai-2024-06-20-thai)| +|[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04][sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]|Chinese| 支持多种方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/telespeech/models.html#sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04)| + +
+ +### Useful links + +- Documentation: https://k2-fsa.github.io/sherpa/onnx/ +- Bilibili 演示视频: https://search.bilibili.com/all?keyword=%E6%96%B0%E4%B8%80%E4%BB%A3Kaldi + +### How to reach us + +Please see +https://k2-fsa.github.io/sherpa/social-groups.html +for 新一代 Kaldi **微信交流群** and **QQ 交流群**. + +## Projects using sherpa-onnx + +### [Sherpa Voice / @siteed/sherpa-onnx.rn](https://github.com/deeeed/audiolab) + +> React Native wrapper and demo app for validating sherpa-onnx on iOS, +> Android, and Web, including ASR, TTS, VAD, KWS, speaker ID, diarization, +> language ID, punctuation, audio tagging, and speech enhancement. + +- [NPM package](https://www.npmjs.com/package/@siteed/sherpa-onnx.rn) +- [Live demo](https://deeeed.github.io/audiolab/sherpa-voice/) + +### [Speed of Sound](https://github.com/zugaldia/speedofsound) + +> A voice-typing application for the Linux desktop (GTK4/Adwaita). +> It captures microphone audio, transcribes it offline using Sherpa ONNX ASR models, +> optionally polishes the text with an LLM, and types the result into the active window +> via XDG Remote Desktop Portal keyboard simulation. + +### [VoxSherpa TTS](https://github.com/CodeBySonu95/VoxSherpa-TTS) + +> VoxSherpa TTS is a 100% offline Android Text-to-Speech app powered by Sherpa-ONNX. +> It supports Kokoro-82M, Piper, and VITS engines with multilingual support including +> Hindi, English, British English, Japanese, Chinese and 50+ more languages. + +- [Download APK (All Versions)](https://github.com/CodeBySonu95/VoxSherpa-TTS/releases) +- Android 11+ · 100% offline · No telemetry + +
+ +| Generate | Models | Library | Settings | +|:---:|:---:|:---:|:---:| +| | | | | + +
+ +--- +### [BreezeApp](https://github.com/mtkresearch/BreezeApp) from [MediaTek Research](https://github.com/mtkresearch) + +> BreezeAPP is a mobile AI application developed for both Android and iOS platforms. +> Users can download it directly from the App Store and enjoy a variety of features +> offline, including speech-to-text, text-to-speech, text-based chatbot interactions, +> and image question-answering + + - [Download APK for BreezeAPP](https://huggingface.co/MediaTek-Research/BreezeApp/resolve/main/BreezeApp.apk) + - [APK 中国镜像](https://hf-mirror.com/MediaTek-Research/BreezeApp/blob/main/BreezeApp.apk) + +| 1 | 2 | 3 | +|---|---|---| +|![](https://github.com/user-attachments/assets/1cdbc057-b893-4de6-9e9c-f1d7dfd1d992)|![](https://github.com/user-attachments/assets/d77cd98e-b057-442f-860d-d5befd5c769b)|![](https://github.com/user-attachments/assets/57e546bf-3d39-45b9-b392-b48ca4fb3c58)| + +### [Open-LLM-VTuber](https://github.com/t41372/Open-LLM-VTuber) + +Talk to any LLM with hands-free voice interaction, voice interruption, and Live2D taking +face running locally across platforms + +See also + +### [voiceapi](https://github.com/ruzhila/voiceapi) + +
+ Streaming ASR and TTS based on FastAPI + + +It shows how to use the ASR and TTS Python APIs with FastAPI. +
+ +### [腾讯会议摸鱼工具 TMSpeech](https://github.com/jxlpzqc/TMSpeech) + +Uses streaming ASR in C# with graphical user interface. + +Video demo in Chinese: [【开源】Windows实时字幕软件（网课/开会必备）](https://www.bilibili.com/video/BV1rX4y1p7Nx) + +### [lol互动助手](https://github.com/l1veIn/lol-wom-electron) + +It uses the JavaScript API of sherpa-onnx along with [Electron](https://electronjs.org/) + +Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率的英雄联盟工具！英雄联盟的最后一块拼图！和游戏中的每个人无障碍沟通！](https://www.bilibili.com/video/BV142tje9E74) + +### [Sherpa-ONNX 语音识别服务器](https://github.com/hfyydd/sherpa-onnx-server) + +A server based on nodejs providing Restful API for speech recognition. + +### [QSmartAssistant](https://github.com/xinhecuican/QSmartAssistant) + +一个模块化，全过程可离线，低占用率的对话机器人/智能音箱 + +It uses QT. Both [ASR](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#asr) +and [TTS](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#tts) +are used. + +### [Flutter-EasySpeechRecognition](https://github.com/Jason-chen-coder/Flutter-EasySpeechRecognition) + +It extends [./flutter-examples/streaming_asr](./flutter-examples/streaming_asr) by +downloading models inside the app to reduce the size of the app. + +Note: [[Team B] Sherpa AI backend](https://github.com/umgc/spring2025/pull/82) also uses +sherpa-onnx in a Flutter APP. + +### [sherpa-onnx-unity](https://github.com/xue-fei/sherpa-onnx-unity) + +sherpa-onnx in Unity. See also [#1695](https://github.com/k2-fsa/sherpa-onnx/issues/1695), +[#1892](https://github.com/k2-fsa/sherpa-onnx/issues/1892), and [#1859](https://github.com/k2-fsa/sherpa-onnx/issues/1859) + +### [xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) + +本项目为xiaozhi-esp32提供后端服务，帮助您快速搭建ESP32设备控制服务器 +Backend service for xiaozhi-esp32, helps you quickly build an ESP32 device control server. + +See also + + - [ASR新增轻量级sherpa-onnx-asr](https://github.com/xinnan-tech/xiaozhi-esp32-server/issues/315) + - [feat: ASR增加sherpa-onnx模型](https://github.com/xinnan-tech/xiaozhi-esp32-server/pull/379) + +### [KaithemAutomation](https://github.com/EternityForest/KaithemAutomation) + +Pure Python, GUI-focused home automation/consumer grade SCADA. + +It uses TTS from sherpa-onnx. See also [✨ Speak command that uses the new globally configured TTS model.](https://github.com/EternityForest/KaithemAutomation/commit/8e64d2b138725e426532f7d66bb69dd0b4f53693) + +### [Open-XiaoAI KWS](https://github.com/idootop/open-xiaoai-kws) + +Enable custom wake word for XiaoAi Speakers. 让小爱音箱支持自定义唤醒词。 + +Video demo in Chinese: [小爱同学启动～˶╹ꇴ╹˶！](https://www.bilibili.com/video/BV1YfVUz5EMj) + +### [C++ WebSocket ASR Server](https://github.com/mawwalker/stt-server) + +It provides a WebSocket server based on C++ for ASR using sherpa-onnx. + +### [Go WebSocket Server](https://github.com/bbeyondllove/asr_server) + +It provides a WebSocket server based on the Go programming language for sherpa-onnx. + +### [Making robot Paimon, Ep10 "The AI Part 1"](https://www.youtube.com/watch?v=KxPKkwxGWZs) + +It is a [YouTube video](https://www.youtube.com/watch?v=KxPKkwxGWZs), +showing how the author tried to use AI so he can have a conversation with Paimon. + +It uses sherpa-onnx for speech-to-text and text-to-speech. +|1| +|---| +|![](https://github.com/user-attachments/assets/f6eea2d5-1807-42cb-9160-be8da2971e1f)| + +### [TtsReader - Desktop application](https://github.com/ys-pro-duction/TtsReader) + +A desktop text-to-speech application built using Kotlin Multiplatform. + +### [MentraOS](https://github.com/Mentra-Community/MentraOS) + +> Smart glasses OS, with dozens of built-in apps. Users get AI assistant, notifications, +> translation, screen mirror, captions, and more. Devs get to write 1 app that runs on +> any pair of smart glasses. + +It uses sherpa-onnx for real-time speech recognition on iOS and Android devices. +See also + +It uses Swift for iOS and Java for Android. + +### [flet_sherpa_onnx](https://github.com/SamYuan1990/flet_sherpa_onnx) + +Flet ASR/STT component based on sherpa-onnx. +Example [a chat box agent](https://github.com/SamYuan1990/i18n-agent-action) + +### [achatbot-go](https://github.com/ai-bot-pro/achatbot-go) + +a multimodal chatbot based on go with sherpa-onnx's speech lib api. + +### [fcitx5-vinput](https://github.com/xifan2333/fcitx5-vinput) + +Local offline voice input plugin for [Fcitx5](https://github.com/fcitx/fcitx5) (Linux input method framework). +It uses C++ with offline ASR for speech recognition, supporting push-to-talk, +command mode, and optional LLM post-processing. + +Video demo in Chinese: [fcitx5-vinput](https://www.bilibili.com/video/BV1a6cUzVE6F) + +### [Wake Word](https://github.com/analyticsinmotion/wake-word) + +A VS Code extension for hands-free voice-activated coding. It uses sherpa-onnx for real-time +keyword spotting (KWS) to detect custom wake phrases and trigger VS Code commands by voice. +Audio capture is handled by [decibri](https://github.com/analyticsinmotion/decibri), a +cross-platform Node.js microphone streaming library with prebuilt native binaries. + +- [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=analytics-in-motion.wake-word) +- [Open VSX](https://open-vsx.org/extension/analytics-in-motion/wake-word) +- [decibri integration guides for sherpa-onnx](https://decibri.dev/docs/node/integrations/sherpa-onnx-stt.html) + +### [SmartSub](https://github.com/buxuku/SmartSub) + +> SmartSub is a local-first cross-platform desktop application for the complete subtitle production pipeline: audio/video transcription, subtitle translation, proofreading, and subtitle burning/muxing. +> +> It natively integrates sherpa-onnx to power three offline ASR engines — FunASR, Qwen3-ASR, and FireRedASR — delivering high-accuracy Chinese and multilingual speech recognition entirely on-device, with no file uploads required. + +[silero-vad]: https://github.com/snakers4/silero-vad +[Raspberry Pi]: https://www.raspberrypi.com/ +[RV1126]: https://www.rock-chips.com/uploads/pdf/2022.8.26/191/RV1126%20Brief%20Datasheet.pdf +[LicheePi4A]: https://sipeed.com/licheepi4a +[VisionFive 2]: https://www.starfivetech.com/en/site/boards +[旭日X3派]: https://developer.horizon.ai/api/v1/fileData/documents_pi/index.html +[爱芯派]: https://wiki.sipeed.com/hardware/zh/maixIII/ax-pi/axpi.html +[hf-space-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/speaker-diarization +[hf-space-speaker-diarization-cn]: https://hf.qhduan.com/spaces/k2-fsa/speaker-diarization +[hf-space-asr]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition +[hf-space-asr-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition +[Whisper]: https://github.com/openai/whisper +[hf-space-asr-whisper]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-asr-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-tts]: https://huggingface.co/spaces/k2-fsa/text-to-speech +[hf-space-tts-cn]: https://hf.qhduan.com/spaces/k2-fsa/text-to-speech +[hf-space-subtitle]: https://huggingface.co/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-subtitle-cn]: https://hf.qhduan.com/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-audio-tagging]: https://huggingface.co/spaces/k2-fsa/audio-tagging +[hf-space-audio-tagging-cn]: https://hf.qhduan.com/spaces/k2-fsa/audio-tagging +[hf-space-source-separation]: https://huggingface.co/spaces/k2-fsa/source-separation +[hf-space-source-separation-cn]: https://hf.qhduan.com/spaces/k2-fsa/source-separation +[hf-space-slid-whisper]: https://huggingface.co/spaces/k2-fsa/spoken-language-identification +[hf-space-slid-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/spoken-language-identification +[wasm-hf-vad]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-sherpa-onnx +[wasm-ms-vad]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-sherpa-onnx +[wasm-hf-streaming-asr-zh-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-ms-streaming-asr-zh-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-hf-streaming-asr-zh-en-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-streaming-asr-zh-en-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[Paraformer-large]: https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary +[wasm-hf-streaming-asr-zh-en-yue-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-ms-streaming-asr-zh-en-yue-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-hf-streaming-asr-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-en +[wasm-ms-streaming-asr-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-en +[SenseVoice]: https://github.com/FunAudioLLM/SenseVoice +[wasm-hf-vad-asr-zh-zipformer-ctc-07-03]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc +[wasm-ms-vad-asr-zh-zipformer-ctc-07-03]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/summary +[wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-ja-ko-cantonese-sense-voice +[wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-en-jp-ko-cantonese-sense-voice +[wasm-hf-vad-asr-en-whisper-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-ms-vad-asr-en-whisper-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-hf-vad-asr-en-moonshine-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-ms-vad-asr-en-moonshine-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-hf-vad-asr-en-zipformer-gigaspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-ms-vad-asr-en-zipformer-gigaspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-hf-vad-asr-zh-zipformer-wenetspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[wasm-ms-vad-asr-zh-zipformer-wenetspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[reazonspeech]: https://research.reazon.jp/_static/reazonspeech_nlp2023.pdf +[wasm-hf-vad-asr-ja-zipformer-reazonspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[wasm-ms-vad-asr-ja-zipformer-reazonspeech]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[gigaspeech2]: https://github.com/speechcolab/gigaspeech2 +[wasm-hf-vad-asr-th-zipformer-gigaspeech2]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[wasm-ms-vad-asr-th-zipformer-gigaspeech2]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[telespeech-asr]: https://github.com/tele-ai/telespeech-asr +[wasm-hf-vad-asr-zh-telespeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-ms-vad-asr-zh-telespeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-hf-vad-asr-zh-en-paraformer-large]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-vad-asr-zh-en-paraformer-large]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-hf-vad-asr-zh-en-paraformer-small]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[wasm-ms-vad-asr-zh-en-paraformer-small]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[dolphin]: https://github.com/dataoceanai/dolphin +[wasm-ms-vad-asr-multi-lang-dolphin-base]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc +[wasm-hf-vad-asr-multi-lang-dolphin-base]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc + +[wasm-hf-tts-matcha-zh-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-matcha +[wasm-hf-tts-matcha-zh]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-tts-matcha +[wasm-ms-tts-matcha-zh-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-matcha +[wasm-ms-tts-matcha-zh]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-tts-matcha +[wasm-hf-tts-matcha-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-matcha +[wasm-ms-tts-matcha-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-matcha +[wasm-hf-tts-piper-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-ms-tts-piper-en]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-hf-tts-piper-de]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-ms-tts-piper-de]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-hf-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx +[wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx +[wasm-hf-voice-cloning-zipvoice]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-zipvoice +[wasm-ms-voice-cloning-zipvoice]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-zipvoice +[wasm-hf-voice-cloning-pocket]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-pocket +[wasm-ms-voice-cloning-pocket]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-pocket +[apk-speaker-diarization]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk.html +[apk-speaker-diarization-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk-cn.html +[apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html +[apk-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-cn.html +[apk-simula-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr.html +[apk-simula-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr-cn.html +[apk-tts]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine.html +[apk-tts-cn]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine-cn.html +[apk-vad]: https://k2-fsa.github.io/sherpa/onnx/vad/apk.html +[apk-vad-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-cn.html +[apk-vad-asr]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr.html +[apk-vad-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr-cn.html +[apk-2pass]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass.html +[apk-2pass-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass-cn.html +[apk-at]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk.html +[apk-at-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-cn.html +[apk-at-wearos]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos.html +[apk-at-wearos-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos-cn.html +[apk-sid]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk.html +[apk-sid-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk-cn.html +[apk-slid]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk.html +[apk-slid-cn]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk-cn.html +[apk-kws]: https://k2-fsa.github.io/sherpa/onnx/kws/apk.html +[apk-kws-cn]: https://k2-fsa.github.io/sherpa/onnx/kws/apk-cn.html +[apk-flutter-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[apk-flutter-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[flutter-tts-android]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android.html +[flutter-tts-android-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android-cn.html +[flutter-tts-linux]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux.html +[flutter-tts-linux-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux-cn.html +[flutter-tts-macos-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64.html +[flutter-tts-macos-arm64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64-cn.html +[flutter-tts-macos-arm64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64.html +[flutter-tts-macos-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64-cn.html +[flutter-tts-win-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win.html +[flutter-tts-win-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win-cn.html +[lazarus-subtitle]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles.html +[lazarus-subtitle-cn]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles-cn.html +[asr-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +[tts-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +[vad-models]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +[kws-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models +[at-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/audio-tagging-models +[sid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[slid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[punct-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models +[speaker-segmentation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +[GigaSpeech]: https://github.com/SpeechColab/GigaSpeech +[WenetSpeech]: https://github.com/wenet-e2e/WenetSpeech +[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-korean-2024-06-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2 +[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01.tar.bz2 +[sherpa-onnx-zipformer-ru-2024-09-18]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ru-2024-09-18.tar.bz2 +[sherpa-onnx-zipformer-korean-2024-06-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-korean-2024-06-24.tar.bz2 +[sherpa-onnx-zipformer-thai-2024-06-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-thai-2024-06-20.tar.bz2 +[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-paraformer-zh-2024-03-09]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2 +[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +[sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 +[Moonshine tiny]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +[NVIDIA Jetson Orin NX]: https://developer.download.nvidia.com/assets/embedded/secure/jetson/orin_nx/docs/Jetson_Orin_NX_DS-10712-001_v0.5.pdf?RCPGu9Q6OVAOv7a7vgtwc9-BLScXRIWq6cSLuditMALECJ_dOj27DgnqAPGVnT2VpiNpQan9SyFy-9zRykR58CokzbXwjSA7Gj819e91AXPrWkGZR3oS1VLxiDEpJa_Y0lr7UT-N4GnXtb8NlUkP4GkCkkF_FQivGPrAucCUywL481GH_WpP_p7ziHU1Wg==&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLmhrLyJ9 +[NVIDIA Jetson Nano B01]: https://www.seeedstudio.com/blog/2020/01/16/new-revision-of-jetson-nano-dev-kit-now-supports-new-jetson-nano-module/ +[speech-enhancement-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models +[source-separation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/source-separation-models +[RK3588]: https://www.rock-chips.com/uploads/pdf/2022.8.26/192/RK3588%20Brief%20Datasheet.pdf +[spleeter]: https://github.com/deezer/spleeter +[UVR]: https://github.com/Anjok07/ultimatevocalremovergui +[gtcrn]: https://github.com/Xiaobin-Rong/gtcrn +[tts-url]: https://k2-fsa.github.io/sherpa/onnx/tts/all-in-one.html +[ss-url]: https://k2-fsa.github.io/sherpa/onnx/source-separation/index.html +[sd-url]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/index.html +[slid-url]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/index.html +[at-url]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/index.html +[vad-url]: https://k2-fsa.github.io/sherpa/onnx/vad/index.html +[kws-url]: https://k2-fsa.github.io/sherpa/onnx/kws/index.html +[punct-url]: https://k2-fsa.github.io/sherpa/onnx/punctuation/index.html +[se-url]: https://k2-fsa.github.io/sherpa/onnx/speech-enhancement/index.html +[rknpu-doc]: https://k2-fsa.github.io/sherpa/onnx/rknn/index.html +[qnn-doc]: https://k2-fsa.github.io/sherpa/onnx/qnn/index.html +[ascend-doc]: https://k2-fsa.github.io/sherpa/onnx/ascend/index.html +[axera-npu]: https://axera-tech.com/Skill/166.html +[SpacemiT-K1]: https://cdn-resource.spacemit.com/file/chip/K1/K1_brief_zh.pdf +[SpacemiT-K3]: https://cdn-resource.spacemit.com/file/chip/K3/K3_brief_zh.pdf diff --git a/crates/sherpa-onnx-sys/build.rs b/crates/sherpa-onnx-sys/build.rs new file mode 100644 index 0000000000..274dfb293e --- /dev/null +++ b/crates/sherpa-onnx-sys/build.rs @@ -0,0 +1,475 @@ +use std::env; +use std::error::Error; +use std::ffi::OsStr; +use std::fs; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; +use std::{collections::HashSet, ffi::OsString}; + +use bzip2::read::BzDecoder; +use tar::Archive; + +const RELEASE_BASE_URL: &str = "https://github.com/k2-fsa/sherpa-onnx/releases/download"; +const SHERPA_ONNX_STATIC_LIBS: &[&str] = &[ + "sherpa-onnx-c-api", + "sherpa-onnx-core", + "kaldi-decoder-core", + "sherpa-onnx-kaldifst-core", + "sherpa-onnx-fstfar", + "sherpa-onnx-fst", + "kaldi-native-fbank-core", + "kissfft-float", + "piper_phonemize", + "espeak-ng", + "ucd", + "onnxruntime", + "ssentencepiece_core", +]; + +type DynError = Box; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LinkMode { + Static, + Shared, +} + +fn main() { + if let Err(err) = try_main() { + panic!("{err}"); + } +} + +fn try_main() -> Result<(), DynError> { + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_LIB_DIR"); + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_ARCHIVE_DIR"); + println!("cargo:rerun-if-env-changed=DOCS_RS"); + + if env::var_os("DOCS_RS").is_some() { + // docs.rs sets DOCS_RS=1; skip downloading/linking native libraries + // so that `cargo doc` can succeed without the real C artifacts. + return Ok(()); + } + + let target_os = env::var("CARGO_CFG_TARGET_OS")?; + let target_arch = env::var("CARGO_CFG_TARGET_ARCH")?; + let link_mode = resolve_link_mode()?; + let lib_dir = resolve_lib_dir(link_mode, &target_os, &target_arch)?; + + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + + if link_mode == LinkMode::Shared && matches!(target_os.as_str(), "linux" | "macos") { + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + emit_relative_rpath(&target_os); + copy_unix_runtime_libs(&lib_dir, &target_os)?; + } + + if link_mode == LinkMode::Shared && target_os == "windows" { + copy_windows_runtime_dlls(&lib_dir)?; + } + + match (link_mode, target_os.as_str()) { + (LinkMode::Static, "ios") => emit_ios_link_directives(), + (LinkMode::Shared, "android") => emit_android_link_directives(), + (LinkMode::Static, _) => emit_static_link_directives(&target_os), + (LinkMode::Shared, _) => emit_shared_link_directives(), + } + + Ok(()) +} + +fn resolve_link_mode() -> Result { + let static_enabled = env::var_os("CARGO_FEATURE_STATIC").is_some(); + let shared_enabled = env::var_os("CARGO_FEATURE_SHARED").is_some(); + + if static_enabled && shared_enabled { + return Err("Features `static` and `shared` cannot be enabled at the same time".into()); + } + + if shared_enabled { + Ok(LinkMode::Shared) + } else { + Ok(LinkMode::Static) + } +} + +fn resolve_lib_dir( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + if let Some(path) = env::var_os("SHERPA_ONNX_LIB_DIR") { + let path = PathBuf::from(path); + if !path.is_dir() { + return Err(format!( + "SHERPA_ONNX_LIB_DIR does not exist or is not a directory: {}", + path.display() + ) + .into()); + } + return Ok(path); + } + + if matches!(target_os, "android" | "ios") { + return Err(format!( + "SHERPA_ONNX_LIB_DIR must point at the verified sherpa-onnx v{} \ + mobile libraries when building for os={target_os}, arch={target_arch}", + env!("CARGO_PKG_VERSION") + ) + .into()); + } + + download_prebuilt_libs(link_mode, target_os, target_arch) +} + +fn download_prebuilt_libs( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let archive_name = archive_name(link_mode, target_os, target_arch)?; + let archive_stem = archive_name.trim_end_matches(".tar.bz2"); + + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let cache_root = target_dir_from_out_dir(&out_dir)?.join("sherpa-onnx-prebuilt"); + let extracted_dir = cache_root.join(archive_stem); + let lib_dir = extracted_dir.join("lib"); + + if lib_dir.is_dir() { + return Ok(lib_dir); + } + + fs::create_dir_all(&cache_root)?; + + let archive_path = cache_root.join(&archive_name); + if !archive_path.is_file() { + if let Some(local_archive_dir) = env::var_os("SHERPA_ONNX_ARCHIVE_DIR") { + let local_archive_path = PathBuf::from(local_archive_dir).join(&archive_name); + if !local_archive_path.is_file() { + return Err(format!( + "SHERPA_ONNX_ARCHIVE_DIR does not contain expected archive: {}", + local_archive_path.display() + ) + .into()); + } + + copy_file_atomically(&local_archive_path, &archive_path)?; + } else { + let version = env!("CARGO_PKG_VERSION"); + let url = format!("{RELEASE_BASE_URL}/v{version}/{archive_name}"); + eprintln!("Downloading sherpa-onnx libs from {url}"); + + let response = ureq::builder() + .try_proxy_from_env(true) + .build() + .get(&url) + .call() + .map_err(|e| format!("Failed to download sherpa-onnx archive from {url}: {e}"))?; + let mut reader = response.into_reader(); + write_reader_atomically(&mut reader, &archive_path)?; + } + } + + if extracted_dir.exists() { + fs::remove_dir_all(&extracted_dir)?; + } + + let unpack_result: Result<(), DynError> = (|| { + let tar_file = File::open(&archive_path)?; + let decoder = BzDecoder::new(tar_file); + let mut archive = Archive::new(decoder); + archive.unpack(&cache_root)?; + Ok(()) + })(); + if let Err(err) = unpack_result { + let _ = fs::remove_file(&archive_path); + let _ = fs::remove_dir_all(&extracted_dir); + return Err(format!( + "Failed to unpack cached archive {}: {err}", + archive_path.display() + ) + .into()); + } + + if !lib_dir.is_dir() { + return Err(format!( + "Downloaded archive did not contain a lib directory: {}", + lib_dir.display() + ) + .into()); + } + + eprintln!("Downloaded sherpa-onnx libs to {}", extracted_dir.display()); + + Ok(lib_dir) +} + +fn archive_name( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let version = env!("CARGO_PKG_VERSION"); + let name = match (link_mode, target_os, target_arch) { + (LinkMode::Static, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-static-lib.tar.bz2") + } + (LinkMode::Static, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-static-MT-Release-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-shared-cpu-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-shared-MT-Release-lib.tar.bz2") + } + _ => { + return Err(format!( + "Unsupported target for sherpa-onnx prebuilt libs: os={target_os}, arch={target_arch}" + ) + .into()) + } + }; + + Ok(name) +} + +fn emit_shared_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); +} + +fn emit_android_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=log"); +} + +fn emit_ios_link_directives() { + println!("cargo:rustc-link-lib=static=sherpa-onnx"); + println!("cargo:rustc-link-lib=static=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=c++"); + for framework in [ + "Accelerate", + "CoreML", + "Foundation", + "Metal", + "QuartzCore", + "Security", + ] { + println!("cargo:rustc-link-lib=framework={framework}"); + } +} + +fn emit_static_link_directives(target_os: &str) { + for lib in SHERPA_ONNX_STATIC_LIBS { + println!("cargo:rustc-link-lib=static={lib}"); + } + + match target_os { + "linux" => { + println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-link-lib=dylib=m"); + println!("cargo:rustc-link-lib=dylib=pthread"); + println!("cargo:rustc-link-lib=dylib=dl"); + } + "macos" => { + println!("cargo:rustc-link-lib=dylib=c++"); + println!("cargo:rustc-link-lib=framework=Foundation"); + } + _ => {} + } +} + +fn target_dir_from_out_dir(out_dir: &Path) -> Result { + if let Ok(explicit_target_dir) = env::var("CARGO_TARGET_DIR") { + return Ok(PathBuf::from(explicit_target_dir)); + } + + if let Some(target_dir) = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new("target"))) + { + return Ok(target_dir.to_path_buf()); + } + + Ok(out_dir.to_path_buf()) +} + +fn emit_relative_rpath(target_os: &str) { + match target_os { + "linux" => println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN"), + "macos" => println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path"), + _ => {} + } +} + +fn profile_output_dirs() -> Result<[PathBuf; 2], DynError> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let profile = env::var("PROFILE")?; + let profile_dir = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new(&profile))) + .ok_or_else(|| { + format!( + "Could not locate Cargo profile directory from {}", + out_dir.display() + ) + })? + .to_path_buf(); + + Ok([profile_dir.clone(), profile_dir.join("examples")]) +} + +fn copy_unix_runtime_libs(lib_dir: &Path, target_os: &str) -> Result<(), DynError> { + let runtime_libs: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| { + path.file_name() + .and_then(OsStr::to_str) + .map(|name| match target_os { + "linux" => name.contains(".so"), + "macos" => name.ends_with(".dylib"), + _ => false, + }) + .unwrap_or(false) + }) + .collect(); + + if runtime_libs.is_empty() { + return Err(format!("No shared runtime libraries found in {}", lib_dir.display()).into()); + } + + let mut copy_plan = Vec::<(PathBuf, OsString)>::new(); + let mut planned_names = HashSet::::new(); + + for lib in runtime_libs { + if !lib.exists() { + continue; + } + + let lib_name = lib + .file_name() + .ok_or_else(|| format!("Invalid runtime library path: {}", lib.display()))? + .to_os_string(); + + let source = fs::canonicalize(&lib).unwrap_or(lib.clone()); + if planned_names.insert(lib_name.clone()) { + copy_plan.push((source.clone(), lib_name)); + } + + if let Some(source_name) = source.file_name() { + let source_name = source_name.to_os_string(); + if planned_names.insert(source_name.clone()) { + copy_plan.push((source.clone(), source_name)); + } + } + } + + if copy_plan.is_empty() { + return Err(format!( + "No usable shared runtime libraries found in {}", + lib_dir.display() + ) + .into()); + } + + for dest_dir in profile_output_dirs()? { + fs::create_dir_all(&dest_dir)?; + for (source, dest_name) in ©_plan { + let dest = dest_dir.join(dest_name); + fs::copy(source, &dest)?; + } + } + + Ok(()) +} + +fn temp_path_for(path: &Path) -> PathBuf { + let mut temp_name = path + .file_name() + .map(OsStr::to_os_string) + .unwrap_or_else(|| OsString::from("tmp")); + temp_name.push(".part"); + path.with_file_name(temp_name) +} + +fn copy_file_atomically(src: &Path, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + fs::copy(src, &temp_path)?; + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn write_reader_atomically(reader: &mut dyn io::Read, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + + { + let mut file = File::create(&temp_path)?; + io::copy(reader, &mut file)?; + file.sync_all()?; + } + + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn copy_windows_runtime_dlls(lib_dir: &Path) -> Result<(), DynError> { + let dlls: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.extension() == Some(OsStr::new("dll"))) + .collect(); + + if dlls.is_empty() { + println!( + "cargo:warning=No runtime DLLs found in {}", + lib_dir.display() + ); + return Ok(()); + } + + let [profile_dir, examples_dir] = profile_output_dirs()?; + for dest_dir in [profile_dir.clone(), examples_dir] { + fs::create_dir_all(&dest_dir)?; + for dll in &dlls { + let dest = dest_dir.join( + dll.file_name() + .ok_or_else(|| format!("Invalid DLL path: {}", dll.display()))?, + ); + fs::copy(dll, &dest)?; + } + } + + println!( + "cargo:warning=Copied Windows runtime DLLs to {} and {}/examples", + profile_dir.display(), + profile_dir.display() + ); + + Ok(()) +} diff --git a/crates/sherpa-onnx-sys/src/audio_tagging.rs b/crates/sherpa-onnx-sys/src/audio_tagging.rs new file mode 100644 index 0000000000..6edf2d5d0a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/audio_tagging.rs @@ -0,0 +1,60 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerAudioTaggingModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingModelConfig { + pub zipformer: OfflineZipformerAudioTaggingModelConfig, + pub ced: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingConfig { + pub model: AudioTaggingModelConfig, + pub labels: *const c_char, + pub top_k: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioEvent { + pub name: *const c_char, + pub index: i32, + pub prob: c_float, +} + +#[repr(C)] +pub struct AudioTagging { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateAudioTagging(config: *const AudioTaggingConfig) -> *const AudioTagging; + + pub fn SherpaOnnxDestroyAudioTagging(tagger: *const AudioTagging); + + pub fn SherpaOnnxAudioTaggingCreateOfflineStream( + tagger: *const AudioTagging, + ) -> *const crate::offline_asr::OfflineStream; + + pub fn SherpaOnnxAudioTaggingCompute( + tagger: *const AudioTagging, + s: *const crate::offline_asr::OfflineStream, + top_k: i32, + ) -> *const *const AudioEvent; + + pub fn SherpaOnnxAudioTaggingFreeResults(p: *const *const AudioEvent); +} diff --git a/crates/sherpa-onnx-sys/src/kws.rs b/crates/sherpa-onnx-sys/src/kws.rs new file mode 100644 index 0000000000..25a2dd26c8 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/kws.rs @@ -0,0 +1,88 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordResult { + pub keyword: *const c_char, + pub tokens: *const c_char, + pub tokens_arr: *const *const c_char, + pub count: i32, + pub timestamps: *mut c_float, + pub start_time: c_float, + pub json: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordSpotterConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: super::online_asr::OnlineModelConfig, + pub max_active_paths: i32, + pub num_trailing_blanks: i32, + pub keywords_score: c_float, + pub keywords_threshold: c_float, + pub keywords_file: *const c_char, + pub keywords_buf: *const c_char, + pub keywords_buf_size: i32, +} + +#[repr(C)] +pub struct KeywordSpotter { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateKeywordSpotter( + config: *const KeywordSpotterConfig, + ) -> *const KeywordSpotter; + + pub fn SherpaOnnxDestroyKeywordSpotter(spotter: *const KeywordSpotter); + + pub fn SherpaOnnxCreateKeywordStream( + spotter: *const KeywordSpotter, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxCreateKeywordStreamWithKeywords( + spotter: *const KeywordSpotter, + keywords: *const c_char, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxIsKeywordStreamReady( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxResetKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleKeywordStreams( + spotter: *const KeywordSpotter, + streams: *const *const super::online_asr::OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetKeywordResult( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const KeywordResult; + + pub fn SherpaOnnxDestroyKeywordResult(r: *const KeywordResult); + + pub fn SherpaOnnxGetKeywordResultAsJson( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxFreeKeywordResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/lib.rs b/crates/sherpa-onnx-sys/src/lib.rs new file mode 100644 index 0000000000..b4161cc616 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/lib.rs @@ -0,0 +1,42 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +extern "C" { + pub fn SherpaOnnxGetVersionStr() -> *const c_char; + pub fn SherpaOnnxGetGitSha1() -> *const c_char; + pub fn SherpaOnnxGetGitDate() -> *const c_char; + pub fn SherpaOnnxFileExists(filename: *const c_char) -> i32; +} + +pub mod audio_tagging; +pub mod kws; +pub mod offline_asr; +pub mod offline_punctuation; +pub mod offline_speaker_diarization; +pub mod online_asr; +pub mod online_punctuation; +pub mod resampler; +pub mod speaker_embedding; +pub mod speech_denoiser; +pub mod spoken_language_identification; +pub mod tts; +pub mod vad; +pub mod wave; + +pub use audio_tagging::*; +pub use kws::*; +pub use offline_asr::*; +pub use offline_punctuation::*; +pub use offline_speaker_diarization::*; +pub use online_asr::*; +pub use online_punctuation::*; +pub use resampler::*; +pub use speaker_embedding::*; +pub use speech_denoiser::*; +pub use spoken_language_identification::*; +pub use tts::*; +pub use vad::*; +pub use wave::*; diff --git a/crates/sherpa-onnx-sys/src/offline_asr.rs b/crates/sherpa-onnx-sys/src/offline_asr.rs new file mode 100644 index 0000000000..9d0d28c748 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_asr.rs @@ -0,0 +1,281 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineParaformerModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineNemoEncDecCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWhisperModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub task: *const c_char, + pub tail_paddings: i32, + pub enable_token_timestamps: i32, + pub enable_segment_timestamps: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCanaryModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub src_lang: *const c_char, + pub tgt_lang: *const c_char, + pub use_pnc: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMoonshineModelConfig { + pub preprocessor: *const c_char, + pub encoder: *const c_char, + pub uncached_decoder: *const c_char, + pub cached_decoder: *const c_char, + pub merged_decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTdnnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineLMConfig { + pub model: *const c_char, + pub scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSenseVoiceModelConfig { + pub model: *const c_char, + pub language: *const c_char, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineDolphinModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWenetCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineOmnilingualAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFunASRNanoModelConfig { + pub encoder_adaptor: *const c_char, + pub llm: *const c_char, + pub embedding: *const c_char, + pub tokenizer: *const c_char, + pub system_prompt: *const c_char, + pub user_prompt: *const c_char, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub language: *const c_char, + pub itn: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineQwen3ASRModelConfig { + pub conv_frontend: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tokenizer: *const c_char, + pub max_total_len: i32, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCohereTranscribeModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub use_punct: i32, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineModelConfig { + pub transducer: OfflineTransducerModelConfig, + pub paraformer: OfflineParaformerModelConfig, + pub nemo_ctc: OfflineNemoEncDecCtcModelConfig, + pub whisper: OfflineWhisperModelConfig, + pub tdnn: OfflineTdnnModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub model_type: *const c_char, + pub modeling_unit: *const c_char, + pub bpe_vocab: *const c_char, + pub telespeech_ctc: *const c_char, + + pub sense_voice: OfflineSenseVoiceModelConfig, + pub moonshine: OfflineMoonshineModelConfig, + pub fire_red_asr: OfflineFireRedAsrModelConfig, + pub dolphin: OfflineDolphinModelConfig, + pub zipformer_ctc: OfflineZipformerCtcModelConfig, + pub canary: OfflineCanaryModelConfig, + pub wenet_ctc: OfflineWenetCtcModelConfig, + pub omnilingual: OfflineOmnilingualAsrCtcModelConfig, + pub medasr: OfflineMedAsrCtcModelConfig, + pub funasr_nano: OfflineFunASRNanoModelConfig, + pub fire_red_asr_ctc: OfflineFireRedAsrCtcModelConfig, + pub qwen3_asr: OfflineQwen3ASRModelConfig, + pub cohere_transcribe: OfflineCohereTranscribeModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineRecognizerConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: OfflineModelConfig, + pub lm_config: OfflineLMConfig, + + pub decoding_method: *const c_char, + pub max_active_paths: i32, + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + pub blank_penalty: c_float, + pub hr: super::online_asr::HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OfflineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineRecognizer( + config: *const OfflineRecognizerConfig, + ) -> *const OfflineRecognizer; + + pub fn SherpaOnnxDestroyOfflineRecognizer(recognizer: *const OfflineRecognizer); + + pub fn SherpaOnnxCreateOfflineStream( + recognizer: *const OfflineRecognizer, + ) -> *const OfflineStream; + + pub fn SherpaOnnxCreateOfflineStreamWithHotwords( + recognizer: *const OfflineRecognizer, + hotwords: *const c_char, + ) -> *const OfflineStream; + + pub fn SherpaOnnxDestroyOfflineStream(stream: *const OfflineStream); + + pub fn SherpaOnnxAcceptWaveformOffline( + stream: *const OfflineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxOfflineStreamSetOption( + stream: *const OfflineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOfflineStreamGetOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOfflineStreamHasOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> i32; + + pub fn SherpaOnnxDecodeOfflineStream( + recognizer: *const OfflineRecognizer, + stream: *const OfflineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOfflineStreams( + recognizer: *const OfflineRecognizer, + streams: *const *const OfflineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOfflineStreamResultAsJson(stream: *const OfflineStream) -> *const c_char; + + pub fn SherpaOnnxDestroyOfflineStreamResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_punctuation.rs b/crates/sherpa-onnx-sys/src/offline_punctuation.rs new file mode 100644 index 0000000000..06547846cf --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_punctuation.rs @@ -0,0 +1,40 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationModelConfig { + pub ct_transformer: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationConfig { + pub model: OfflinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OfflinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflinePunctuation( + config: *const OfflinePunctuationConfig, + ) -> *const OfflinePunctuation; + + pub fn SherpaOnnxDestroyOfflinePunctuation(punct: *const OfflinePunctuation); + + pub fn SherpaOfflinePunctuationAddPunct( + punct: *const OfflinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOfflinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs new file mode 100644 index 0000000000..00232e0a98 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs @@ -0,0 +1,98 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationPyannoteModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationModelConfig { + pub pyannote: OfflineSpeakerSegmentationPyannoteModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FastClusteringConfig { + pub num_clusters: i32, + pub threshold: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationConfig { + pub segmentation: OfflineSpeakerSegmentationModelConfig, + pub embedding: crate::speaker_embedding::SpeakerEmbeddingExtractorConfig, + pub clustering: FastClusteringConfig, + pub min_duration_on: c_float, + pub min_duration_off: c_float, +} + +#[repr(C)] +pub struct OfflineSpeakerDiarization { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineSpeakerDiarizationResult { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationSegment { + pub start: c_float, + pub end: c_float, + pub speaker: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeakerDiarization( + config: *const OfflineSpeakerDiarizationConfig, + ) -> *const OfflineSpeakerDiarization; + + pub fn SherpaOnnxDestroyOfflineSpeakerDiarization(sd: *const OfflineSpeakerDiarization); + + pub fn SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + sd: *const OfflineSpeakerDiarization, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationSetConfig( + sd: *const OfflineSpeakerDiarization, + config: *const OfflineSpeakerDiarizationConfig, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + r: *const OfflineSpeakerDiarizationResult, + ) -> *const OfflineSpeakerDiarizationSegment; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + s: *const OfflineSpeakerDiarizationSegment, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationProcess( + sd: *const OfflineSpeakerDiarization, + samples: *const c_float, + n: i32, + ) -> *const OfflineSpeakerDiarizationResult; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + r: *const OfflineSpeakerDiarizationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/online_asr.rs b/crates/sherpa-onnx-sys/src/online_asr.rs new file mode 100644 index 0000000000..522f934d79 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_asr.rs @@ -0,0 +1,202 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineParaformerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineZipformer2CtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineNemoCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineToneCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineModelConfig { + pub transducer: OnlineTransducerModelConfig, + pub paraformer: OnlineParaformerModelConfig, + pub zipformer2_ctc: OnlineZipformer2CtcModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + + pub model_type: *const c_char, + + // cjkchar | bpe | cjkchar+bpe + pub modeling_unit: *const c_char, + + pub bpe_vocab: *const c_char, + + pub tokens_buf: *const u8, + pub tokens_buf_size: i32, + + pub nemo_ctc: OnlineNemoCtcModelConfig, + pub t_one_ctc: OnlineToneCtcModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FeatureConfig { + pub sample_rate: i32, + pub feature_dim: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineCtcFstDecoderConfig { + pub graph: *const c_char, + pub max_active: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct HomophoneReplacerConfig { + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub rule_fsts: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineRecognizerConfig { + pub feat_config: FeatureConfig, + pub model_config: OnlineModelConfig, + + // greedy_search | modified_beam_search + pub decoding_method: *const c_char, + + pub max_active_paths: i32, + + pub enable_endpoint: i32, + + pub rule1_min_trailing_silence: c_float, + pub rule2_min_trailing_silence: c_float, + pub rule3_min_utterance_length: c_float, + + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + + pub ctc_fst_decoder_config: OnlineCtcFstDecoderConfig, + + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + + pub blank_penalty: c_float, + + pub hotwords_buf: *const u8, + pub hotwords_buf_size: i32, + + pub hr: HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OnlineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlineRecognizer( + config: *const OnlineRecognizerConfig, + ) -> *const OnlineRecognizer; + + pub fn SherpaOnnxDestroyOnlineRecognizer(recognizer: *const OnlineRecognizer); + + pub fn SherpaOnnxCreateOnlineStream(recognizer: *const OnlineRecognizer) + -> *const OnlineStream; + + pub fn SherpaOnnxCreateOnlineStreamWithHotwords( + recognizer: *const OnlineRecognizer, + hotwords: *const c_char, + ) -> *const OnlineStream; + + pub fn SherpaOnnxDestroyOnlineStream(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamAcceptWaveform( + stream: *const OnlineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxIsOnlineStreamReady( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeOnlineStream( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOnlineStreams( + recognizer: *const OnlineRecognizer, + streams: *const *const OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOnlineStreamResultAsJson( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxDestroyOnlineStreamResultJson(s: *const c_char); + + pub fn SherpaOnnxOnlineStreamReset( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxOnlineStreamInputFinished(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamSetOption( + stream: *const OnlineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOnlineStreamGetOption( + stream: *const OnlineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlineStreamHasOption(stream: *const OnlineStream, key: *const c_char) -> i32; + + pub fn SherpaOnnxOnlineStreamIsEndpoint( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/online_punctuation.rs b/crates/sherpa-onnx-sys/src/online_punctuation.rs new file mode 100644 index 0000000000..1ff1fce1b0 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_punctuation.rs @@ -0,0 +1,41 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationModelConfig { + pub cnn_bilstm: *const c_char, + pub bpe_vocab: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationConfig { + pub model: OnlinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OnlinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlinePunctuation( + config: *const OnlinePunctuationConfig, + ) -> *const OnlinePunctuation; + + pub fn SherpaOnnxDestroyOnlinePunctuation(punctuation: *const OnlinePunctuation); + + pub fn SherpaOnnxOnlinePunctuationAddPunct( + punctuation: *const OnlinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/resampler.rs b/crates/sherpa-onnx-sys/src/resampler.rs new file mode 100644 index 0000000000..fd49835662 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/resampler.rs @@ -0,0 +1,42 @@ +use std::os::raw::c_float; + +#[repr(C)] +pub struct SherpaOnnxLinearResampler { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SherpaOnnxResampleOut { + pub samples: *const f32, + pub n: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateLinearResampler( + samp_rate_in_hz: i32, + samp_rate_out_hz: i32, + filter_cutoff_hz: c_float, + num_zeros: i32, + ) -> *const SherpaOnnxLinearResampler; + + pub fn SherpaOnnxDestroyLinearResampler(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerReset(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerResample( + p: *const SherpaOnnxLinearResampler, + input: *const f32, + input_dim: i32, + flush: i32, + ) -> *const SherpaOnnxResampleOut; + + pub fn SherpaOnnxLinearResamplerResampleFree(p: *const SherpaOnnxResampleOut); + + pub fn SherpaOnnxLinearResamplerResampleGetInputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; + + pub fn SherpaOnnxLinearResamplerResampleGetOutputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/speaker_embedding.rs b/crates/sherpa-onnx-sys/src/speaker_embedding.rs new file mode 100644 index 0000000000..ca745e3106 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speaker_embedding.rs @@ -0,0 +1,131 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingExtractorConfig { + pub model: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpeakerEmbeddingExtractor { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeakerEmbeddingManager { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerSpeakerMatch { + pub score: c_float, + pub name: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerBestMatchesResult { + pub matches: *const SpeakerEmbeddingManagerSpeakerMatch, + pub count: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateSpeakerEmbeddingExtractor( + config: *const SpeakerEmbeddingExtractorConfig, + ) -> *const SpeakerEmbeddingExtractor; + + pub fn SherpaOnnxDestroySpeakerEmbeddingExtractor(p: *const SpeakerEmbeddingExtractor); + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDim(p: *const SpeakerEmbeddingExtractor) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorCreateStream( + p: *const SpeakerEmbeddingExtractor, + ) -> *const crate::online_asr::OnlineStream; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorIsReady( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> *const c_float; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(v: *const c_float); + + pub fn SherpaOnnxCreateSpeakerEmbeddingManager(dim: i32) -> *const SpeakerEmbeddingManager; + + pub fn SherpaOnnxDestroySpeakerEmbeddingManager(p: *const SpeakerEmbeddingManager); + + pub fn SherpaOnnxSpeakerEmbeddingManagerAdd( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddList( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddListFlattened( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + n: i32, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerRemove( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerSearch( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + ) -> *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeSearch(name: *const c_char); + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetBestMatches( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + n: i32, + ) -> *const SpeakerEmbeddingManagerBestMatchesResult; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeBestMatches( + r: *const SpeakerEmbeddingManagerBestMatchesResult, + ); + + pub fn SherpaOnnxSpeakerEmbeddingManagerVerify( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + threshold: c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerContains( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(p: *const SpeakerEmbeddingManager) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetAllSpeakers( + p: *const SpeakerEmbeddingManager, + ) -> *const *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeAllSpeakers(names: *const *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/speech_denoiser.rs b/crates/sherpa-onnx-sys/src/speech_denoiser.rs new file mode 100644 index 0000000000..dcf7f1e0e4 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speech_denoiser.rs @@ -0,0 +1,88 @@ +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserGtcrnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserDpdfNetModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserModelConfig { + pub gtcrn: OfflineSpeechDenoiserGtcrnModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct DenoisedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +pub struct OfflineSpeechDenoiser { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineSpeechDenoiser { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeechDenoiser( + config: *const OfflineSpeechDenoiserConfig, + ) -> *const OfflineSpeechDenoiser; + pub fn SherpaOnnxDestroyOfflineSpeechDenoiser(p: *const OfflineSpeechDenoiser); + pub fn SherpaOnnxOfflineSpeechDenoiserGetSampleRate(p: *const OfflineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOfflineSpeechDenoiserRun( + p: *const OfflineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + + pub fn SherpaOnnxCreateOnlineSpeechDenoiser( + config: *const OnlineSpeechDenoiserConfig, + ) -> *const OnlineSpeechDenoiser; + pub fn SherpaOnnxDestroyOnlineSpeechDenoiser(p: *const OnlineSpeechDenoiser); + pub fn SherpaOnnxOnlineSpeechDenoiserGetSampleRate(p: *const OnlineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserGetFrameShiftInSamples( + p: *const OnlineSpeechDenoiser, + ) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserRun( + p: *const OnlineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserFlush( + p: *const OnlineSpeechDenoiser, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserReset(p: *const OnlineSpeechDenoiser); + + pub fn SherpaOnnxDestroyDenoisedAudio(audio: *const DenoisedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/spoken_language_identification.rs b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs new file mode 100644 index 0000000000..f9d8f46753 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs @@ -0,0 +1,54 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationWhisperConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tail_paddings: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationConfig { + pub whisper: SpokenLanguageIdentificationWhisperConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpokenLanguageIdentification { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationResult { + pub lang: *const c_char, +} + +extern "C" { + pub fn SherpaOnnxCreateSpokenLanguageIdentification( + config: *const SpokenLanguageIdentificationConfig, + ) -> *const SpokenLanguageIdentification; + + pub fn SherpaOnnxDestroySpokenLanguageIdentification(slid: *const SpokenLanguageIdentification); + + pub fn SherpaOnnxSpokenLanguageIdentificationCreateOfflineStream( + slid: *const SpokenLanguageIdentification, + ) -> *const super::offline_asr::OfflineStream; + + pub fn SherpaOnnxSpokenLanguageIdentificationCompute( + slid: *const SpokenLanguageIdentification, + stream: *const super::offline_asr::OfflineStream, + ) -> *const SpokenLanguageIdentificationResult; + + pub fn SherpaOnnxDestroySpokenLanguageIdentificationResult( + r: *const SpokenLanguageIdentificationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/tts.rs b/crates/sherpa-onnx-sys/src/tts.rs new file mode 100644 index 0000000000..6be479c56a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/tts.rs @@ -0,0 +1,172 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float, c_void}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsVitsModelConfig { + pub model: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub noise_scale_w: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsMatchaModelConfig { + pub acoustic_model: *const c_char, + pub vocoder: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKokoroModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub lang: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKittenModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsZipvoiceModelConfig { + pub tokens: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub vocoder: *const c_char, + pub data_dir: *const c_char, + pub lexicon: *const c_char, + pub feat_scale: c_float, + pub t_shift: c_float, + pub target_rms: c_float, + pub guidance_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsPocketModelConfig { + pub lm_flow: *const c_char, + pub lm_main: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub text_conditioner: *const c_char, + pub vocab_json: *const c_char, + pub token_scores_json: *const c_char, + pub voice_embedding_cache_capacity: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsSupertonicModelConfig { + pub duration_predictor: *const c_char, + pub text_encoder: *const c_char, + pub vector_estimator: *const c_char, + pub vocoder: *const c_char, + pub tts_json: *const c_char, + pub unicode_indexer: *const c_char, + pub voice_style: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsModelConfig { + pub vits: OfflineTtsVitsModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub matcha: OfflineTtsMatchaModelConfig, + pub kokoro: OfflineTtsKokoroModelConfig, + pub kitten: OfflineTtsKittenModelConfig, + pub zipvoice: OfflineTtsZipvoiceModelConfig, + pub pocket: OfflineTtsPocketModelConfig, + pub supertonic: OfflineTtsSupertonicModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsConfig { + pub model: OfflineTtsModelConfig, + pub rule_fsts: *const c_char, + pub max_num_sentences: i32, + pub rule_fars: *const c_char, + pub silence_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGeneratedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGenerationConfig { + pub silence_scale: c_float, + pub speed: c_float, + pub sid: i32, + pub reference_audio: *const f32, + pub reference_audio_len: i32, + pub reference_sample_rate: i32, + pub reference_text: *const c_char, + pub num_steps: i32, + pub extra: *const c_char, +} + +pub type SherpaOnnxGeneratedAudioProgressCallbackWithArg = Option< + unsafe extern "C" fn(samples: *const f32, n: i32, progress: c_float, arg: *mut c_void) -> i32, +>; + +#[repr(C)] +pub struct SherpaOnnxOfflineTts { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineTts( + config: *const OfflineTtsConfig, + ) -> *const SherpaOnnxOfflineTts; + + pub fn SherpaOnnxDestroyOfflineTts(tts: *const SherpaOnnxOfflineTts); + + pub fn SherpaOnnxOfflineTtsSampleRate(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsNumSpeakers(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsGenerateWithConfig( + tts: *const SherpaOnnxOfflineTts, + text: *const c_char, + config: *const SherpaOnnxGenerationConfig, + callback: SherpaOnnxGeneratedAudioProgressCallbackWithArg, + arg: *mut c_void, + ) -> *const SherpaOnnxGeneratedAudio; + + pub fn SherpaOnnxDestroyOfflineTtsGeneratedAudio(p: *const SherpaOnnxGeneratedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/vad.rs b/crates/sherpa-onnx-sys/src/vad.rs new file mode 100644 index 0000000000..b16b21deb9 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/vad.rs @@ -0,0 +1,85 @@ +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +pub struct SileroVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct TenVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct VadModelConfig { + pub silero_vad: SileroVadModelConfig, + pub sample_rate: i32, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + pub ten_vad: TenVadModelConfig, +} + +#[repr(C)] +pub struct CircularBuffer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeechSegment { + pub start: i32, + pub samples: *mut f32, + pub n: i32, +} + +#[repr(C)] +pub struct VoiceActivityDetector { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateCircularBuffer(capacity: i32) -> *const CircularBuffer; + pub fn SherpaOnnxDestroyCircularBuffer(buffer: *const CircularBuffer); + pub fn SherpaOnnxCircularBufferPush(buffer: *const CircularBuffer, p: *const f32, n: i32); + pub fn SherpaOnnxCircularBufferGet( + buffer: *const CircularBuffer, + start_index: i32, + n: i32, + ) -> *const f32; + pub fn SherpaOnnxCircularBufferFree(p: *const f32); + pub fn SherpaOnnxCircularBufferPop(buffer: *const CircularBuffer, n: i32); + pub fn SherpaOnnxCircularBufferSize(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferHead(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferReset(buffer: *const CircularBuffer); + + pub fn SherpaOnnxCreateVoiceActivityDetector( + config: *const VadModelConfig, + buffer_size_in_seconds: c_float, + ) -> *const VoiceActivityDetector; + pub fn SherpaOnnxDestroyVoiceActivityDetector(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorAcceptWaveform( + p: *const VoiceActivityDetector, + samples: *const f32, + n: i32, + ); + pub fn SherpaOnnxVoiceActivityDetectorEmpty(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorDetected(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorPop(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorClear(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFront( + p: *const VoiceActivityDetector, + ) -> *const SpeechSegment; + pub fn SherpaOnnxDestroySpeechSegment(p: *const SpeechSegment); + pub fn SherpaOnnxVoiceActivityDetectorReset(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFlush(p: *const VoiceActivityDetector); +} diff --git a/crates/sherpa-onnx-sys/src/wave.rs b/crates/sherpa-onnx-sys/src/wave.rs new file mode 100644 index 0000000000..c3c8f860fe --- /dev/null +++ b/crates/sherpa-onnx-sys/src/wave.rs @@ -0,0 +1,30 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxWave { + /// Samples normalized to [-1, 1] + pub samples: *const f32, + pub sample_rate: i32, + pub num_samples: i32, +} + +extern "C" { + /// Read a WAV file. Returns NULL on error. + pub fn SherpaOnnxReadWave(filename: *const c_char) -> *const SherpaOnnxWave; + + /// Free memory allocated by SherpaOnnxReadWave + pub fn SherpaOnnxFreeWave(wave: *const SherpaOnnxWave); + + /// Write a WAV file. Returns 1 on success, 0 on failure. + pub fn SherpaOnnxWriteWave( + samples: *const f32, + n: i32, + sample_rate: i32, + filename: *const c_char, + ) -> i32; +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 074b8f739e..7d87c64c8d 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -75,6 +75,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if 1.0.4", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -694,6 +708,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -718,6 +738,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bip39" version = "2.2.2" @@ -1023,6 +1049,7 @@ dependencies = [ "buzz-media", "buzz-persona", "buzz-sdk", + "buzz-voice", "bytes", "bzip2 0.6.1", "chrono", @@ -1143,6 +1170,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "ort", + "ort-sys", + "rand 0.10.2", + "sentencepiece-model", + "serde", + "serde_json", + "sherpa-onnx", + "tokenizers", +] + [[package]] name = "by_address" version = "1.2.1" @@ -1545,6 +1586,7 @@ dependencies = [ "itoa", "rustversion", "ryu", + "serde", "static_assertions", ] @@ -1796,6 +1838,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -2120,6 +2172,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -2617,6 +2678,12 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "euclid" version = "0.22.14" @@ -2675,6 +2742,17 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-srgb8" version = "1.0.0" @@ -4746,6 +4824,39 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.118", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loom" version = "0.7.2" @@ -4855,6 +4966,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "markup5ever" version = "0.38.0" @@ -4881,6 +5008,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-async" version = "0.2.11" @@ -4980,7 +5117,7 @@ dependencies = [ "mesh-llm-types", "model-artifact", "nostr-sdk", - "prost", + "prost 0.14.4", "rand 0.10.2", "rustls", "serde", @@ -5119,7 +5256,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "rand 0.10.2", "regex-lite", "reqwest 0.12.28", @@ -5207,8 +5344,8 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=f455d493a2ae82baf2a32 dependencies = [ "anyhow", "async-trait", - "prost", - "prost-build", + "prost 0.14.4", + "prost-build 0.14.4", "protoc-bin-vendored", "rmcp", "schemars 1.2.1", @@ -5243,7 +5380,7 @@ dependencies = [ "anyhow", "hex", "iroh", - "prost", + "prost 0.14.4", "serde_json", "sha2 0.10.9", ] @@ -5358,6 +5495,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if 1.0.4", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "mime" version = "0.3.17" @@ -5503,6 +5662,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "more-asserts" version = "0.3.1" @@ -5630,6 +5811,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -6611,7 +6807,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "reqwest 0.12.28", "thiserror 2.0.18", ] @@ -6626,7 +6822,7 @@ dependencies = [ "const-hex", "opentelemetry", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "serde", "serde_json", "tonic", @@ -6692,6 +6888,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + [[package]] name = "os_pipe" version = "1.2.3" @@ -6919,6 +7133,16 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap 2.14.0", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -7183,6 +7407,15 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.1" @@ -7393,6 +7626,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.4" @@ -7400,7 +7643,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.4", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.118", + "tempfile", ] [[package]] @@ -7413,15 +7676,28 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", - "prost", - "prost-types", + "prost 0.14.4", + "prost-types 0.14.4", "regex", "syn 2.0.118", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "prost-derive" version = "0.14.4" @@ -7435,13 +7711,35 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "prost-reflect" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" +dependencies = [ + "logos", + "miette", + "once_cell", + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost", + "prost 0.14.4", ] [[package]] @@ -7508,6 +7806,33 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "protox" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" +dependencies = [ + "bytes", + "miette", + "prost 0.13.5", + "prost-reflect", + "prost-types 0.13.5", + "protox-parse", + "thiserror 1.0.69", +] + +[[package]] +name = "protox-parse" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" +dependencies = [ + "logos", + "miette", + "prost-types 0.13.5", + "thiserror 1.0.69", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -7756,7 +8081,7 @@ dependencies = [ "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7819,7 +8144,7 @@ dependencies = [ "strum", "time", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7828,6 +8153,43 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "realfft" version = "3.5.0" @@ -8640,6 +9002,18 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "sentencepiece-model" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b87bf750a8322c3236d7aa63c1f4a6862187d00d2d8b038e1dfe263bfe43ec" +dependencies = [ + "miette", + "prost 0.13.5", + "prost-build 0.13.5", + "protox", +] + [[package]] name = "serde" version = "1.0.228" @@ -9072,8 +9446,8 @@ name = "skippy-protocol" version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=f455d493a2ae82baf2a326e2d0fda351433b4b30#f455d493a2ae82baf2a326e2d0fda351433b4b30" dependencies = [ - "prost", - "prost-build", + "prost 0.14.4", + "prost-build 0.14.4", "protoc-bin-vendored", "serde", ] @@ -9252,6 +9626,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sse-stream" version = "0.2.4" @@ -9633,7 +10019,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -10208,7 +10594,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "bitflags 2.13.0", - "fancy-regex", + "fancy-regex 0.11.0", "filedescriptor", "finl_unicode", "fixedbitset 0.4.2", @@ -10371,6 +10757,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str 0.9.1", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -10704,7 +11123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", - "prost", + "prost 0.14.4", "tonic", ] @@ -10885,7 +11304,7 @@ checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6" dependencies = [ "memchr", "nom 8.0.0", - "petgraph", + "petgraph 0.8.3", ] [[package]] @@ -11056,6 +11475,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -11070,9 +11498,15 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ "itertools", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -11085,6 +11519,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index d689544688..d3c52235fd 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -89,6 +89,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" } buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" } buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" } buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" } +buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" } iroh = { version = "1.0.2", optional = true } mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true } mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true } diff --git a/desktop/src-tauri/examples/pocket_bench.rs b/desktop/src-tauri/examples/pocket_bench.rs deleted file mode 100644 index b4f5635a95..0000000000 --- a/desktop/src-tauri/examples/pocket_bench.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Cold-vs-warm latency bench for Pocket TTS. -//! -//! This duplicates the small config-building snippet from `huddle::pocket` so it -//! doesn't depend on changing module visibility for a one-off dev tool. -//! Keep in sync with `huddle::pocket::load_text_to_speech`. -//! -//! Run with the model files in a directory (defaults to /tmp/pocket-tts-bench): -//! cargo run --release --example pocket_bench -//! cargo run --release --example pocket_bench /path/to/pocket-tts - -use std::path::PathBuf; -use std::time::Instant; - -use sherpa_onnx::{ - self, GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsModelConfig, - OfflineTtsPocketModelConfig, Wave, -}; - -const SAMPLE_RATE: u32 = 24_000; -const TEST_TEXT: &str = - "Hello, this is a test of the new Pocket TTS engine running on sherpa-onnx."; - -fn main() { - let model_dir = std::env::args() - .nth(1) - .unwrap_or_else(|| "/tmp/pocket-tts-bench".to_string()); - println!("Model dir: {model_dir}"); - - let dir = PathBuf::from(&model_dir); - let p = |name: &str| dir.join(name).to_string_lossy().into_owned(); - - let t0 = Instant::now(); - let cfg = OfflineTtsConfig { - model: OfflineTtsModelConfig { - pocket: OfflineTtsPocketModelConfig { - lm_main: Some(p("lm_main.int8.onnx")), - lm_flow: Some(p("lm_flow.int8.onnx")), - encoder: Some(p("encoder.onnx")), - decoder: Some(p("decoder.int8.onnx")), - text_conditioner: Some(p("text_conditioner.onnx")), - vocab_json: Some(p("vocab.json")), - token_scores_json: Some(p("token_scores.json")), - voice_embedding_cache_capacity: 16, - }, - num_threads: 1, - debug: false, - ..Default::default() - }, - ..Default::default() - }; - let engine = OfflineTts::create(&cfg).expect("engine create"); - let load_ms = t0.elapsed().as_secs_f32() * 1000.0; - println!("Engine load: {load_ms:.1} ms"); - - let t0 = Instant::now(); - let voice_path = dir.join("reference_sample.wav"); - let wave = Wave::read(voice_path.to_str().unwrap()).expect("voice WAV"); - let samples = wave.samples().to_vec(); - let sr = wave.sample_rate(); - let voice_ms = t0.elapsed().as_secs_f32() * 1000.0; - println!("Voice load: {voice_ms:.1} ms"); - - let gen = || GenerationConfig { - speed: 1.05, - num_steps: 1, - silence_scale: 1.0, // production setting (huddle::pocket::SYNTH_SILENCE_SCALE) - reference_audio: Some(samples.clone()), - reference_sample_rate: sr, - ..Default::default() - }; - - let t0 = Instant::now(); - let cold = engine - .generate_with_config(TEST_TEXT, &gen(), None:: bool>) - .expect("cold synth"); - let cold_ms = t0.elapsed().as_secs_f32() * 1000.0; - let cold_audio_ms = (cold.samples().len() as f32 / SAMPLE_RATE as f32) * 1000.0; - let cold_rtf_x = cold_audio_ms / cold_ms; - println!( - "Cold synth: {cold_ms:.1} ms → {cold_audio_ms:.1} ms audio → {cold_rtf_x:.2}× realtime" - ); - - let t0 = Instant::now(); - let warm = engine - .generate_with_config(TEST_TEXT, &gen(), None:: bool>) - .expect("warm synth"); - let warm_ms = t0.elapsed().as_secs_f32() * 1000.0; - let warm_audio_ms = (warm.samples().len() as f32 / SAMPLE_RATE as f32) * 1000.0; - let warm_rtf_x = warm_audio_ms / warm_ms; - println!( - "Warm synth: {warm_ms:.1} ms → {warm_audio_ms:.1} ms audio → {warm_rtf_x:.2}× realtime" - ); - - let out_path = "/tmp/pocket_bench_out.wav"; - let ok = sherpa_onnx::write(out_path, warm.samples(), SAMPLE_RATE as i32); - println!( - "Wrote {} ({} samples, ok={ok})", - out_path, - warm.samples().len() - ); - - let delta_ms = cold_ms - warm_ms; - let delta_pct = (delta_ms / warm_ms) * 100.0; - println!(); - println!("Cold/warm delta: {delta_ms:+.1} ms ({delta_pct:+.1}%)"); - println!( - "Decision: warmup {}.", - if delta_ms > 200.0 { - "RECOMMENDED — significant cold-call penalty" - } else if delta_ms > 50.0 { - "OPTIONAL — small cold-call penalty" - } else { - "UNNECESSARY — cold and warm essentially equal" - } - ); -} diff --git a/desktop/src-tauri/examples/pocket_clip_probe.rs b/desktop/src-tauri/examples/pocket_clip_probe.rs deleted file mode 100644 index ad8657599f..0000000000 --- a/desktop/src-tauri/examples/pocket_clip_probe.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Clipping probe for any fixed playback gain applied after Pocket TTS synth. -//! -//! Synthesises a spread of sentences (short/long, calm/energetic) and reports -//! the raw peak of each, the post-gain peak, and the fraction of samples that -//! would hit a ±1.0 clamp — i.e. how much a fixed gain would flat-top the -//! waveform ("blown out" distortion). -//! -//! History: the production pipeline briefly shipped a fixed 9.3× gain -//! calibrated on a single bench utterance that peaked at 0.076. This probe -//! showed real output peaks at 0.4–0.97, so that gain clipped 13–34% of all -//! samples (the 2026-06-12 "blown out" report). Production now applies no -//! gain — run this probe before reintroducing one. -//! -//! Run with model files in ~/.buzz/models/pocket-tts (override with arg 1): -//! cargo run --release --example pocket_clip_probe - -use std::path::PathBuf; - -use sherpa_onnx::{ - self, GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsModelConfig, - OfflineTtsPocketModelConfig, Wave, -}; - -/// Candidate gain under test (the regressed production value). -const GAIN: f32 = 9.3; - -const PROMPTS: &[&str] = &[ - "Hello, this is a test of the new Pocket TTS engine running on sherpa-onnx.", - "Yep, I can hear you.", - "Absolutely! That sounds fantastic, let's do it right now!", - "The quick brown fox jumps over the lazy dog near the riverbank.", - "I found three problems in the code: a race condition, a memory leak, and an off-by-one error in the loop bounds.", - "No.", - "Warning! The build failed because seventeen tests crashed unexpectedly!", - "Sure, I can walk you through the whole pipeline step by step whenever you're ready.", -]; - -fn main() { - let model_dir = std::env::args().nth(1).unwrap_or_else(|| { - dirs::home_dir() - .expect("home dir") - .join(".buzz/models/pocket-tts") - .to_string_lossy() - .into_owned() - }); - eprintln!("Model dir: {model_dir}"); - - let dir = PathBuf::from(&model_dir); - let p = |name: &str| dir.join(name).to_string_lossy().into_owned(); - - let cfg = OfflineTtsConfig { - model: OfflineTtsModelConfig { - pocket: OfflineTtsPocketModelConfig { - lm_main: Some(p("lm_main.int8.onnx")), - lm_flow: Some(p("lm_flow.int8.onnx")), - encoder: Some(p("encoder.onnx")), - decoder: Some(p("decoder.int8.onnx")), - text_conditioner: Some(p("text_conditioner.onnx")), - vocab_json: Some(p("vocab.json")), - token_scores_json: Some(p("token_scores.json")), - voice_embedding_cache_capacity: 16, - }, - num_threads: 1, - debug: false, - ..Default::default() - }, - ..Default::default() - }; - let engine = OfflineTts::create(&cfg).expect("engine create"); - - let voice_path = dir.join("reference_sample.wav"); - let wave = Wave::read(voice_path.to_str().unwrap()).expect("voice WAV"); - let voice_samples = wave.samples().to_vec(); - let voice_sr = wave.sample_rate(); - - let gen = || GenerationConfig { - speed: 1.05, - num_steps: 1, - silence_scale: 1.0, - reference_audio: Some(voice_samples.clone()), - reference_sample_rate: voice_sr, - ..Default::default() - }; - - let _ = engine.generate_with_config("warmup.", &gen(), None:: bool>); - - println!( - "{:<46} | {:>8} | {:>9} | {:>9} | {:>10}", - "prompt", "raw peak", "raw RMS", "post-gain", "% clipped" - ); - println!("{}", "-".repeat(95)); - - let mut worst_clip = 0.0f32; - for prompt in PROMPTS { - let out = engine - .generate_with_config(prompt, &gen(), None:: bool>) - .expect("synth"); - let samples = out.samples(); - - let peak = samples.iter().fold(0.0f32, |m, s| m.max(s.abs())); - let rms = (samples.iter().map(|s| s * s).sum::() / samples.len() as f32).sqrt(); - let post = peak * GAIN; - let clipped = samples.iter().filter(|s| s.abs() * GAIN > 1.0).count(); - let clip_pct = 100.0 * clipped as f32 / samples.len() as f32; - worst_clip = worst_clip.max(clip_pct); - - let label: String = prompt.chars().take(44).collect(); - println!("{label:<46} | {peak:>8.4} | {rms:>9.4} | {post:>9.3} | {clip_pct:>9.3}%"); - } - - println!(); - println!( - "Verdict: worst-case clipped fraction {worst_clip:.3}% — {}", - if worst_clip > 0.1 { - "AUDIBLE DISTORTION LIKELY (gain too hot)" - } else if worst_clip > 0.0 { - "marginal — occasional transient clipping" - } else { - "no clipping at this gain" - } - ); -} diff --git a/desktop/src-tauri/examples/pocket_onset_probe.rs b/desktop/src-tauri/examples/pocket_onset_probe.rs deleted file mode 100644 index 05b4d0193c..0000000000 --- a/desktop/src-tauri/examples/pocket_onset_probe.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! Onset-attenuation probe for Pocket TTS. -//! -//! Synthesises a handful of short sentences and dumps per-sentence onset -//! statistics (samples[0], 1ms/5ms/20ms peak + RMS) so we can decide whether -//! the production `apply_fades` 8 ms fade-in is masking real audio. -//! -//! Also writes the raw (un-faded, un-normalised) audio of each sentence to -//! /tmp so they can be inspected in Audacity / aplay without rodio in the -//! loop. -//! -//! Run with model files in /tmp/pocket-tts-bench (override with arg 1): -//! cargo run --release --example pocket_onset_probe -//! cargo run --release --example pocket_onset_probe /path/to/pocket-tts - -use std::path::PathBuf; - -use sherpa_onnx::{ - self, GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsModelConfig, - OfflineTtsPocketModelConfig, Wave, -}; - -const SAMPLE_RATE: u32 = 24_000; - -/// Test prompts chosen to span different onsets: -/// - palatal glide 'Y' (soft onset) -/// - voiceless fricative 'H' (very soft onset) -/// - labio-velar glide 'W' (medium onset) -/// - voiceless stop 'T' (hard onset) -const PROMPTS: &[&str] = &[ - "Yep, I can hear you.", - "Hello there friend.", - "What can I help with?", - "Try this experiment now.", -]; - -fn main() { - let model_dir = std::env::args() - .nth(1) - .unwrap_or_else(|| "/tmp/pocket-tts-bench".to_string()); - eprintln!("Model dir: {model_dir}"); - - let dir = PathBuf::from(&model_dir); - let p = |name: &str| dir.join(name).to_string_lossy().into_owned(); - - let cfg = OfflineTtsConfig { - model: OfflineTtsModelConfig { - pocket: OfflineTtsPocketModelConfig { - lm_main: Some(p("lm_main.int8.onnx")), - lm_flow: Some(p("lm_flow.int8.onnx")), - encoder: Some(p("encoder.onnx")), - decoder: Some(p("decoder.int8.onnx")), - text_conditioner: Some(p("text_conditioner.onnx")), - vocab_json: Some(p("vocab.json")), - token_scores_json: Some(p("token_scores.json")), - voice_embedding_cache_capacity: 16, - }, - num_threads: 1, - debug: false, - ..Default::default() - }, - ..Default::default() - }; - let engine = OfflineTts::create(&cfg).expect("engine create"); - - let voice_path = dir.join("reference_sample.wav"); - let wave = Wave::read(voice_path.to_str().unwrap()).expect("voice WAV"); - let voice_samples = wave.samples().to_vec(); - let voice_sr = wave.sample_rate(); - - // Warmup so we're not measuring cold-call jitter. - { - let cfg = GenerationConfig { - speed: 1.05, - num_steps: 1, - silence_scale: 1.0, // production setting (huddle::pocket::SYNTH_SILENCE_SCALE) - reference_audio: Some(voice_samples.clone()), - reference_sample_rate: voice_sr, - ..Default::default() - }; - let _ = engine.generate_with_config("warmup.", &cfg, None:: bool>); - } - - println!( - "{:<28} | {:>10} | {:>10} {:>10} | {:>10} {:>10} | {:>10} {:>10}", - "prompt", - "samples[0]", - "peak@1ms", - "rms@1ms", - "peak@5ms", - "rms@5ms", - "peak@20ms", - "rms@20ms" - ); - println!("{}", "-".repeat(120)); - - for prompt in PROMPTS { - // Mirror the production prompt-prep (capitalise + terminal punctuation). - // These prompts already have it, so this is just to match what - // sherpa-onnx sees in production. - let cfg = GenerationConfig { - speed: 1.05, - num_steps: 1, - silence_scale: 1.0, // production setting (huddle::pocket::SYNTH_SILENCE_SCALE) - reference_audio: Some(voice_samples.clone()), - reference_sample_rate: voice_sr, - ..Default::default() - }; - let out = engine - .generate_with_config(prompt, &cfg, None:: bool>) - .expect("synth"); - let samples = out.samples(); - - let n_1ms = (SAMPLE_RATE as f32 * 0.001) as usize; - let n_5ms = (SAMPLE_RATE as f32 * 0.005) as usize; - let n_20ms = (SAMPLE_RATE as f32 * 0.020) as usize; - - let stats = |range: &[f32]| -> (f32, f32) { - if range.is_empty() { - return (0.0, 0.0); - } - let peak = range.iter().fold(0.0_f32, |a, &x| a.max(x.abs())); - let sumsq: f32 = range.iter().map(|x| x * x).sum(); - let rms = (sumsq / range.len() as f32).sqrt(); - (peak, rms) - }; - - let first = samples.first().copied().unwrap_or(0.0); - let (p1, r1) = stats(&samples[..n_1ms.min(samples.len())]); - let (p5, r5) = stats(&samples[..n_5ms.min(samples.len())]); - let (p20, r20) = stats(&samples[..n_20ms.min(samples.len())]); - - println!( - "{:<28} | {:>10.6} | {:>10.6} {:>10.6} | {:>10.6} {:>10.6} | {:>10.6} {:>10.6}", - prompt, first, p1, r1, p5, r5, p20, r20 - ); - - let safe: String = prompt - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect(); - let out_path = format!("/tmp/pocket_onset_{}.wav", &safe[..safe.len().min(24)]); - let _ = sherpa_onnx::write(&out_path, samples, SAMPLE_RATE as i32); - eprintln!( - " → wrote {out_path} ({} samples = {:.3} s)", - samples.len(), - samples.len() as f32 / SAMPLE_RATE as f32 - ); - } -} diff --git a/desktop/src-tauri/examples/pocket_quality_ab.rs b/desktop/src-tauri/examples/pocket_quality_ab.rs deleted file mode 100644 index 0c31f1c910..0000000000 --- a/desktop/src-tauri/examples/pocket_quality_ab.rs +++ /dev/null @@ -1,519 +0,0 @@ -//! Reproducible blind Pocket TTS quality corpus generator. -//! -//! Renders Buzz's production prompt preparation and post-processing across: -//! INT8/FP32 × per-sentence/grouped generation. The generated filenames are -//! deterministically blinded; keep `key.json` away from listeners until their -//! scoring sheet is complete. -//! -//! Usage: -//! cargo run --release --example pocket_quality_ab -- \ -//! [--idle-minutes N --only ITEM] -//! -//! The optional idle run intentionally creates one engine per condition, warms -//! all four, sleeps once, and then makes each clip the first generation after -//! dormancy. It requires `--only` because only the first synthesis after an -//! uninterrupted idle is a valid post-idle observation. Run each 5/15-minute -//! item as a separate process. - -// Importing the production module also brings in runtime-only helpers that this -// standalone corpus generator deliberately does not call. -#![allow(dead_code)] - -#[path = "../src/huddle/pocket.rs"] -mod production_pocket; -#[path = "../src/huddle/preprocessing.rs"] -mod production_preprocessing; - -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; - -use serde::Serialize; -use sha2::{Digest, Sha256}; -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -use production_pocket::{prepare_pocket_prompt, SAMPLE_RATE}; -use production_preprocessing::{preprocess_for_tts, split_sentences}; - -const NUM_STEPS: i32 = 1; -const SILENCE_SCALE: f32 = 1.0; -const INTER_SENTENCE_SILENCE_SAMPLES: usize = SAMPLE_RATE as usize / 10; -const LEAD_IN_SAMPLES: usize = SAMPLE_RATE as usize / 50; -const FADE_OUT_SAMPLES: usize = SAMPLE_RATE as usize * 8 / 1000; -const TARGET_RMS_DBFS: f32 = -23.0; -const BLINDING_SEED: &str = "pocket-quality-2026-07-21-v1"; - -const CORPUS: &[CorpusItem] = &[ - CorpusItem { id: "short_one_word", kind: "short", text: "Yep." }, - CorpusItem { id: "short_four_words", kind: "short", text: "Sounds good to me." }, - CorpusItem { - id: "multi_relay_review", - kind: "multi-sentence", - text: "I looked at the relay code this morning. The lease logic is solid. There's one race in the worker claim path, though. I'll write it up and send you a patch.", - }, - CorpusItem { - id: "multi_community_size", - kind: "multi-sentence", - text: "Great question. The answer is it depends on the community size. For small ones, keep it simple.", - }, - CorpusItem { - id: "mixed_agent_message", - kind: "mixed", - text: "That's 42 open PRs right now — mostly small. I'll triage them after lunch.", - }, -]; - -#[derive(Clone, Copy)] -struct CorpusItem { - id: &'static str, - kind: &'static str, - text: &'static str, -} - -#[derive(Clone, Copy, Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum Precision { - Int8, - Fp32, -} - -#[derive(Clone, Copy, Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum Chunking { - PerSentence, - Grouped, -} - -#[derive(Clone, Copy, Debug)] -struct Condition { - precision: Precision, - chunking: Chunking, -} - -const CONDITIONS: [Condition; 4] = [ - Condition { - precision: Precision::Int8, - chunking: Chunking::PerSentence, - }, - Condition { - precision: Precision::Int8, - chunking: Chunking::Grouped, - }, - Condition { - precision: Precision::Fp32, - chunking: Chunking::PerSentence, - }, - Condition { - precision: Precision::Fp32, - chunking: Chunking::Grouped, - }, -]; - -#[derive(Serialize)] -struct KeyFile { - warning: &'static str, - blinding_seed: &'static str, - target_rms_dbfs: f32, - items: Vec, -} - -#[derive(Serialize)] -struct KeyItem { - id: String, - kind: String, - text: String, - clips: Vec, -} - -#[derive(Serialize)] -struct KeyClip { - file: String, - precision: Precision, - chunking: Chunking, - cold_start: bool, - idle_minutes: Option, - synthesis_ms: u128, - audio_seconds: f32, -} - -struct Voice { - samples: Vec, - sample_rate: i32, -} - -struct Engine { - inner: OfflineTts, - voice: Voice, -} - -fn main() -> Result<(), String> { - let mut args = std::env::args().skip(1); - let int8_dir = required_path(args.next(), "INT8 model directory")?; - let fp32_dir = required_path(args.next(), "FP32 model directory")?; - let output_dir = required_path(args.next(), "output directory")?; - let mut idle_minutes = None; - let mut only_item = None; - while let Some(arg) = args.next() { - match arg.as_str() { - "--idle-minutes" => { - idle_minutes = Some( - args.next() - .ok_or("--idle-minutes requires a value")? - .parse::() - .map_err(|e| format!("invalid idle minutes: {e}"))?, - ); - } - "--only" => only_item = Some(args.next().ok_or("--only requires an item ID")?), - _ => return Err(format!("unknown argument: {arg}")), - } - } - - if idle_minutes.is_some() && only_item.is_none() { - return Err("--idle-minutes requires --only so every clip is first-after-idle".into()); - } - if let Some(ref requested) = only_item { - if !CORPUS.iter().any(|item| item.id == requested) { - return Err(format!("unknown corpus item for --only: {requested}")); - } - } - - validate_model_dir(&int8_dir, Precision::Int8)?; - validate_model_dir(&fp32_dir, Precision::Fp32)?; - fs::create_dir_all(&output_dir).map_err(|e| e.to_string())?; - - let mut engines = Vec::with_capacity(CONDITIONS.len()); - for condition in CONDITIONS { - let dir = match condition.precision { - Precision::Int8 => &int8_dir, - Precision::Fp32 => &fp32_dir, - }; - let engine = load_engine(dir, condition.precision)?; - // Production warms once before serving a real utterance. Cold cases use - // separate fresh engines below and deliberately skip this call. - synth_chunks(&engine, &["warmup".to_string()])?; - engines.push(engine); - } - - if let Some(minutes) = idle_minutes { - eprintln!("All four warmed engines idle for {minutes} minute(s)…"); - std::thread::sleep(Duration::from_secs(minutes * 60)); - } - - let mut key_items = Vec::new(); - for item in CORPUS { - if only_item - .as_deref() - .is_some_and(|requested| requested != item.id) - { - continue; - } - let preprocessed = preprocess_for_tts(item.text); - let per_sentence: Vec = split_sentences(&preprocessed) - .into_iter() - .filter(|s| !s.trim().is_empty()) - .collect(); - // These corpus texts are deliberately below the upstream ~50-token - // grouping target, so grouped mode is one exact generate() call. - let grouped = vec![per_sentence.join(" ")]; - let item_dir = output_dir.join(item.id); - fs::create_dir_all(&item_dir).map_err(|e| e.to_string())?; - let clip_order = blinded_order(item.id); - let mut clips = Vec::new(); - - let mut rendered = Vec::new(); - for (condition_index, engine) in engines.iter().enumerate() { - let condition = CONDITIONS[condition_index]; - let chunks = match condition.chunking { - Chunking::PerSentence => &per_sentence, - Chunking::Grouped => &grouped, - }; - let started = Instant::now(); - let audio = synth_chunks(engine, chunks)?; - rendered.push(( - condition_index, - condition, - audio, - started.elapsed().as_millis(), - )); - } - loudness_match_item(&mut rendered); - for (condition_index, condition, audio, synth_ms) in rendered { - let clip_number = clip_order[condition_index] + 1; - let file_name = format!("clip{clip_number}.wav"); - write_wav(&item_dir.join(&file_name), &audio)?; - clips.push(KeyClip { - file: format!("{}/{file_name}", item.id), - precision: condition.precision, - chunking: condition.chunking, - cold_start: false, - idle_minutes, - synthesis_ms: synth_ms, - audio_seconds: audio.len() as f32 / SAMPLE_RATE as f32, - }); - } - clips.sort_by(|a, b| a.file.cmp(&b.file)); - key_items.push(KeyItem { - id: item.id.to_string(), - kind: item.kind.to_string(), - text: item.text.to_string(), - clips, - }); - } - - // Explicit fresh-engine cold-start clips for the two highest-signal texts. - // Idle runs intentionally omit them: they happen after the post-idle clips - // and add no valid idle observation. - for item in if idle_minutes.is_none() { CORPUS } else { &[] } { - if !matches!(item.id, "short_one_word" | "multi_relay_review") { - continue; - } - if only_item - .as_deref() - .is_some_and(|requested| requested != item.id) - { - continue; - } - let cold_id = format!("cold_{}", item.id); - let preprocessed = preprocess_for_tts(item.text); - let sentences: Vec = split_sentences(&preprocessed) - .into_iter() - .filter(|s| !s.trim().is_empty()) - .collect(); - let grouped = vec![sentences.join(" ")]; - let item_dir = output_dir.join(&cold_id); - fs::create_dir_all(&item_dir).map_err(|e| e.to_string())?; - let clip_order = blinded_order(&cold_id); - let mut clips = Vec::new(); - let mut rendered = Vec::new(); - for (condition_index, condition) in CONDITIONS.iter().copied().enumerate() { - let dir = match condition.precision { - Precision::Int8 => &int8_dir, - Precision::Fp32 => &fp32_dir, - }; - let engine = load_engine(dir, condition.precision)?; - let chunks = match condition.chunking { - Chunking::PerSentence => &sentences, - Chunking::Grouped => &grouped, - }; - let started = Instant::now(); - let audio = synth_chunks(&engine, chunks)?; - rendered.push(( - condition_index, - condition, - audio, - started.elapsed().as_millis(), - )); - } - loudness_match_item(&mut rendered); - for (condition_index, condition, audio, synth_ms) in rendered { - let clip_number = clip_order[condition_index] + 1; - let file_name = format!("clip{clip_number}.wav"); - write_wav(&item_dir.join(&file_name), &audio)?; - clips.push(KeyClip { - file: format!("{cold_id}/{file_name}"), - precision: condition.precision, - chunking: condition.chunking, - cold_start: true, - idle_minutes: None, - synthesis_ms: synth_ms, - audio_seconds: audio.len() as f32 / SAMPLE_RATE as f32, - }); - } - clips.sort_by(|a, b| a.file.cmp(&b.file)); - key_items.push(KeyItem { - id: cold_id, - kind: "cold-start".to_string(), - text: item.text.to_string(), - clips, - }); - } - - let key = KeyFile { - warning: "DO NOT OPEN UNTIL LISTENING SCORES ARE FINAL", - blinding_seed: BLINDING_SEED, - target_rms_dbfs: TARGET_RMS_DBFS, - items: key_items, - }; - fs::write( - output_dir.join("key.json"), - serde_json::to_vec_pretty(&key).map_err(|e| e.to_string())?, - ) - .map_err(|e| e.to_string())?; - write_scoring_sheet(&output_dir, &key)?; - println!("Wrote blind corpus to {}", output_dir.display()); - println!("Give listeners the WAV folders and SCORING.md; withhold key.json."); - Ok(()) -} - -fn required_path(value: Option, label: &str) -> Result { - value - .map(PathBuf::from) - .ok_or_else(|| format!("missing {label}")) -} - -fn model_file(precision: Precision, base: &str) -> String { - match precision { - Precision::Int8 => format!("{base}.int8.onnx"), - Precision::Fp32 => format!("{base}.onnx"), - } -} - -fn validate_model_dir(dir: &Path, precision: Precision) -> Result<(), String> { - for file in [ - model_file(precision, "lm_main"), - model_file(precision, "lm_flow"), - "encoder.onnx".into(), - model_file(precision, "decoder"), - "text_conditioner.onnx".into(), - "vocab.json".into(), - "token_scores.json".into(), - "reference_sample.wav".into(), - ] { - if !dir.join(&file).is_file() { - return Err(format!("missing {}", dir.join(file).display())); - } - } - Ok(()) -} - -fn load_engine(dir: &Path, precision: Precision) -> Result { - let p = |name: &str| dir.join(name).to_string_lossy().into_owned(); - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(p(&model_file(precision, "lm_main"))); - cfg.model.pocket.lm_flow = Some(p(&model_file(precision, "lm_flow"))); - cfg.model.pocket.encoder = Some(p("encoder.onnx")); - cfg.model.pocket.decoder = Some(p(&model_file(precision, "decoder"))); - cfg.model.pocket.text_conditioner = Some(p("text_conditioner.onnx")); - cfg.model.pocket.vocab_json = Some(p("vocab.json")); - cfg.model.pocket.token_scores_json = Some(p("token_scores.json")); - cfg.model.pocket.voice_embedding_cache_capacity = 16; - cfg.model.num_threads = 1; - cfg.model.debug = false; - let inner = - OfflineTts::create(&cfg).ok_or_else(|| format!("failed to create {precision:?} engine"))?; - let wave = - Wave::read(&p("reference_sample.wav")).ok_or("failed to read reference_sample.wav")?; - Ok(Engine { - inner, - voice: Voice { - samples: wave.samples().to_vec(), - sample_rate: wave.sample_rate(), - }, - }) -} - -fn synth_chunks(engine: &Engine, chunks: &[String]) -> Result, String> { - let mut out = Vec::new(); - for chunk in chunks { - let prepared = prepare_pocket_prompt(chunk).ok_or("empty prepared prompt")?; - let extra = prepared.max_frames.map(|max_frames| { - HashMap::from([( - "max_frames".to_string(), - serde_json::Value::from(max_frames), - )]) - }); - let cfg = GenerationConfig { - num_steps: NUM_STEPS, - silence_scale: SILENCE_SCALE, - reference_audio: Some(engine.voice.samples.clone()), - reference_sample_rate: engine.voice.sample_rate, - extra, - ..Default::default() - }; - let audio = engine - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| format!("synthesis failed for {chunk:?}"))?; - let mut samples: Vec = audio.samples().iter().map(|s| s.clamp(-1.0, 1.0)).collect(); - apply_fade_out(&mut samples); - out.extend(std::iter::repeat_n(0.0, LEAD_IN_SAMPLES)); - out.extend(samples); - out.extend(std::iter::repeat_n( - 0.0, - INTER_SENTENCE_SILENCE_SAMPLES - LEAD_IN_SAMPLES, - )); - } - Ok(out) -} - -fn apply_fade_out(samples: &mut [f32]) { - let fade = FADE_OUT_SAMPLES.min(samples.len() / 2); - for i in 0..fade { - samples[samples.len() - 1 - i] *= i as f32 / fade as f32; - } -} - -fn active_rms(samples: &[f32]) -> Option { - let (sum_squares, count) = samples - .iter() - .filter(|sample| sample.abs() > 1.0e-4) - .fold((0.0_f32, 0_usize), |(sum, count), sample| { - (sum + sample * sample, count + 1) - }); - (count > 0).then(|| (sum_squares / count as f32).sqrt()) -} - -/// Attenuate every clip in one comparison set to the quietest active-speech RMS. -/// This removes the louder-is-better confound without normalizing dynamics or -/// claiming standards-compliant integrated LUFS. The dBFS value is a ceiling. -fn loudness_match_item(rendered: &mut [(usize, Condition, Vec, u128)]) { - let ceiling = 10.0_f32.powf(TARGET_RMS_DBFS / 20.0); - let target = rendered - .iter() - .filter_map(|(_, _, samples, _)| active_rms(samples)) - .fold(ceiling, f32::min); - for (_, _, samples, _) in rendered { - let Some(rms) = active_rms(samples) else { - continue; - }; - let gain = (target / rms).min(1.0); - for sample in samples { - *sample *= gain; - } - } -} - -fn blinded_order(item_id: &str) -> [usize; 4] { - let mut keyed: Vec<(usize, Vec)> = (0..4) - .map(|index| { - let digest = Sha256::digest(format!("{BLINDING_SEED}:{item_id}:{index}")); - (index, digest.to_vec()) - }) - .collect(); - keyed.sort_by(|a, b| a.1.cmp(&b.1)); - let mut condition_to_clip = [0; 4]; - for (clip, (condition, _)) in keyed.into_iter().enumerate() { - condition_to_clip[condition] = clip; - } - condition_to_clip -} - -fn write_wav(path: &Path, samples: &[f32]) -> Result<(), String> { - let path = path - .to_str() - .ok_or_else(|| format!("non-UTF8 path: {}", path.display()))?; - if sherpa_onnx::write(path, samples, SAMPLE_RATE as i32) { - Ok(()) - } else { - Err(format!("failed to write {path}")) - } -} - -fn write_scoring_sheet(output_dir: &Path, key: &KeyFile) -> Result<(), String> { - let mut sheet = String::from("# Pocket TTS blind listening sheet\n\nDo not open `key.json` until this sheet is complete. Rank best to worst; ties are allowed.\n\n"); - for item in &key.items { - sheet.push_str(&format!( - "## {} ({})\n\n> {}\n\n", - item.id, item.kind, item.text - )); - sheet.push_str("Rank: `____ > ____ > ____ > ____`\n\n| Clip | seam | onset | garble | robotic | timbre | truncate | note |\n|---|---|---|---|---|---|---|---|\n"); - for clip in 1..=4 { - sheet.push_str(&format!( - "| clip{clip} | [ ] | [ ] | [ ] | [ ] | [ ] | [ ] | |\n" - )); - } - sheet.push('\n'); - } - fs::write(output_dir.join("SCORING.md"), sheet).map_err(|e| e.to_string()) -} diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..6df014dc65 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -38,19 +38,17 @@ use sha2::{Digest, Sha256}; /// Computed from a known-good download. Update when upgrading model versions. const STT_ARCHIVE_SHA256: &str = "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"; -/// HuggingFace base URL for the sherpa-onnx Pocket TTS fp32 repackage. +/// HuggingFace base URL for the April Pocket TTS fp32 ONNX bundle. /// -/// Pinned to commit 96d1e53ce3311ca6c2c6a35e2062d36b4cec6fa3 -/// (2026-02-10) for reproducible downloads. -/// -/// fp32 (not int8): a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -/// found the ONNX int8 quantization audibly degraded Pocket TTS output and -/// that fp32 "significantly improved quality even at 1 step". The runtime -/// bundle grows from ~189 MB to ~473 MB; encoder, text conditioner, both -/// JSON tables, and LICENSE are byte-identical between the two repos — only -/// the three quantized sessions (lm_main, lm_flow, decoder) change. +/// Pinned to KevinAHM/pocket-tts-onnx commit +/// 58a6d00cf13d239b6748cb0769f35c580a8f606c (2026-04-21) for reproducible +/// downloads. This is the full-precision `english_2026-04` bundle. const POCKET_HF_BASE: &str = - "https://huggingface.co/csukuangfj2/sherpa-onnx-pocket-tts-2026-01-26/resolve/96d1e53ce3311ca6c2c6a35e2062d36b4cec6fa3"; + "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04"; + +/// The export's CC-BY-4.0 text lives one directory above the language bundle. +const POCKET_LICENSE_URL: &str = + "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/LICENSE"; /// Reference voice WAV: "Mary (f, conversation)" from the Kyutai TTS demo /// voice set — VCTK speaker p333, ai-coustics-enhanced. Pinned to @@ -68,13 +66,14 @@ const POCKET_REFERENCE_WAV_URL: &str = /// Computed from known-good pinned downloads. Update when upgrading model versions. #[rustfmt::skip] const TTS_FILE_HASHES: &[(&str, &str)] = &[ - ("decoder.onnx", "f267880fde6c58b17b0a8f3647eaf8dcfad321f833f32d583ebc2fb2d1a15f10"), - ("encoder.onnx", "e8f2f6d301ffb96e398b138a7dc6d3038622d236044636b73d920bab85890260"), - ("lm_flow.onnx", "79c013a554a54e63319c33c0cc8830cbbedc9b7e448ae7e26f7923ae11f9873e"), - ("lm_main.onnx", "255d1a9263c5abdf36034abfc19c11d21cc5f40f0f87d8361288e972cbd5c578"), - ("text_conditioner.onnx", "0b84e837d7bfaf2c896627b03e3f080320309f37f4fc7df7698c644f7ba5e6b1"), - ("vocab.json", "6fb646346cf931016f70c4921aab0900ce7a304b893cb02135c74e294abfea01"), - ("token_scores.json", "5be2f278caf9b9800741f0fd82bff677f4943ec764c356f907213434b622d958"), + ("bundle.json", "bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f"), + ("bos_before_voice.npy", "f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1"), + ("flow_lm_flow.onnx", "085d239f68897e28fb06e95c743738ad8b8c092ee6dc55f5491313e81ff08062"), + ("flow_lm_main.onnx", "6d18315e2c33ca3e3aa4a4e3dca22f56d007fd823127e24948b37695bf54190f"), + ("mimi_decoder.onnx", "86f038caa02a96a0ff9c25526a0ff43a4906c418197ed72d3e30f720ac7ce802"), + ("mimi_encoder.onnx", "853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d"), + ("text_conditioner.onnx", "4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb"), + ("tokenizer.model", "d461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6"), ("LICENSE", "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6"), ("reference_sample.wav", "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f"), ]; @@ -92,15 +91,9 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ /// honest (each version tag identifies one specific set of model bytes). const STT_MODEL_VERSION: &str = "2"; -/// Model manifest version for Pocket TTS. Increment when upgrading model files. -/// Bumped "1" → "2" when the bundled reference voice changed from KevinAHM's -/// anonymous 16 kHz sample to Mary (VCTK p333, 32 kHz, ai-coustics-enhanced) -/// from kyutai/tts-voices. The hash mismatch on `reference_sample.wav` would -/// fail readiness on its own, but the manifest bump makes the re-download -/// reason explicit and skips the failing-then-re-fetching transient state. -/// Bumped "2" → "3" for the int8 → fp32 model swap (see `POCKET_HF_BASE`): -/// existing int8 installs must re-download the suffixless fp32 sessions. -const TTS_MODEL_VERSION: &str = "3"; +/// Identifies the exact Pocket TTS asset set expected by readiness checks. +/// A mismatch triggers a fresh download and atomic installation. +const TTS_MODEL_VERSION: &str = "4"; /// Filename for the version manifest written alongside model files. const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; @@ -111,7 +104,7 @@ const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; const MAX_STT_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; /// Maximum expected Pocket TTS file size (400 MB per file — largest is -/// `lm_main.onnx` at ~303 MB fp32). +/// `flow_lm_main.onnx` at ~303 MB fp32). const MAX_TTS_FILE_BYTES: u64 = 400 * 1024 * 1024; /// NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx by @@ -181,9 +174,8 @@ Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926). Mimi neural codec by Kyutai is bundled as part of the model. -ONNX export by KevinAHM: https://huggingface.co/KevinAHM/pocket-tts-onnx -Sherpa-onnx repackage by csukuangfj / k2-fsa: -https://huggingface.co/csukuangfj2/sherpa-onnx-pocket-tts-2026-01-26 +April ONNX export by KevinAHM: +https://huggingface.co/KevinAHM/pocket-tts-onnx Bundled reference voice (reference_sample.wav): \"Mary (f, conversation)\" preset from the Kyutai TTS demo voice catalogue @@ -203,13 +195,14 @@ license text for full warranty disclaimer. /// All files that must be present for Pocket TTS to be considered ready. const TTS_EXPECTED_FILES: &[&str] = &[ - "decoder.onnx", - "encoder.onnx", - "lm_flow.onnx", - "lm_main.onnx", + "bundle.json", + "bos_before_voice.npy", + "flow_lm_flow.onnx", + "flow_lm_main.onnx", + "mimi_decoder.onnx", + "mimi_encoder.onnx", "text_conditioner.onnx", - "vocab.json", - "token_scores.json", + "tokenizer.model", "LICENSE", "reference_sample.wav", TTS_LICENSE_FILE_NAME, @@ -453,6 +446,33 @@ impl ModelSlot { self.just_ready.swap(false, Ordering::AcqRel) } + /// Restore the verified backup when an interrupted install leaves the + /// destination absent or incomplete. + fn recover_interrupted_install(&self, models_dir: &Path) { + let final_dir = self.model_dir(models_dir); + let backup_dir = final_dir.with_extension("old"); + if !backup_dir.exists() || self.is_ready(models_dir) { + return; + } + + if final_dir.exists() { + if let Err(error) = std::fs::remove_dir_all(&final_dir) { + eprintln!( + "buzz-desktop: could not remove incomplete {} install: {error}", + self.dir_name + ); + return; + } + } + + if let Err(error) = std::fs::rename(&backup_dir, &final_dir) { + eprintln!( + "buzz-desktop: could not restore interrupted {} install: {error}", + self.dir_name + ); + } + } + /// Spawn a background download task if not already ready or downloading. fn start_download( &self, @@ -561,11 +581,14 @@ impl ModelManager { /// Returns `None` if the home directory cannot be resolved. pub fn new() -> Option { let models_dir = dirs::home_dir()?.join(".buzz").join("models"); - Some(Self { + let manager = Self { models_dir, stt: ModelSlot::new(STT_MODEL_DIR_NAME, STT_EXPECTED_FILES, STT_MODEL_VERSION), tts: ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION), - }) + }; + // Keep the last complete cache usable after an interrupted atomic swap. + manager.tts.recover_interrupted_install(&manager.models_dir); + Some(manager) } // ── STT accessors ──────────────────────────────────────────────────────── @@ -638,7 +661,7 @@ impl ModelManager { } } - /// Start a background Pocket TTS download (~189 MB). No-op if already ready or downloading. + /// Start a background Pocket TTS download. No-op if already ready or downloading. pub fn start_tts_download(&self, http_client: reqwest::Client) { let manager = self.clone(); self.tts.start_download( @@ -755,7 +778,7 @@ impl ModelManager { /// /// Downloads files into `~/.buzz/models/pocket-tts/`: /// - five ONNX sessions (Pocket TTS + Mimi codec) - /// - `vocab.json` / `token_scores.json` for sherpa-onnx text conditioning + /// - April bundle metadata, SentencePiece tokenizer, and learned voice BOS /// - upstream `LICENSE` plus Buzz's `MODEL_LICENSE.txt` attribution sidecar /// - `reference_sample.wav` as the bundled default voice /// @@ -769,19 +792,20 @@ impl ModelManager { fresh_temp_dir(&temp_dir).await?; let model_files = [ - "decoder.onnx", - "encoder.onnx", - "lm_flow.onnx", - "lm_main.onnx", + "bundle.json", + "bos_before_voice.npy", + "flow_lm_flow.onnx", + "flow_lm_main.onnx", + "mimi_decoder.onnx", + "mimi_encoder.onnx", "text_conditioner.onnx", - "vocab.json", - "token_scores.json", - "LICENSE", + "tokenizer.model", ]; let mut downloads: Vec<(String, &'static str)> = model_files .iter() .map(|filename| (format!("{POCKET_HF_BASE}/{filename}"), *filename)) .collect(); + downloads.push((POCKET_LICENSE_URL.to_string(), "LICENSE")); downloads.push((POCKET_REFERENCE_WAV_URL.to_string(), "reference_sample.wav")); let total_files = downloads.len() as u32; @@ -931,24 +955,5 @@ pub fn is_tts_ready() -> bool { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tts_readiness_requires_license_sidecar() { - let temp = tempfile::tempdir().expect("tempdir"); - let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); - let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); - std::fs::create_dir_all(&model_dir).expect("create model dir"); - - for file in TTS_EXPECTED_FILES { - std::fs::write(model_dir.join(file), b"test").expect("write expected file"); - } - std::fs::write(model_dir.join(MANIFEST_FILENAME), TTS_MODEL_VERSION).expect("manifest"); - - assert!(slot.is_ready(temp.path())); - - std::fs::remove_file(model_dir.join(TTS_LICENSE_FILE_NAME)).expect("remove sidecar"); - assert!(!slot.is_ready(temp.path())); - } -} +#[path = "models/tests.rs"] +mod tests; diff --git a/desktop/src-tauri/src/huddle/models/tests.rs b/desktop/src-tauri/src/huddle/models/tests.rs new file mode 100644 index 0000000000..44cd8bc914 --- /dev/null +++ b/desktop/src-tauri/src/huddle/models/tests.rs @@ -0,0 +1,58 @@ +use super::*; + +#[test] +fn tts_readiness_requires_license_sidecar() { + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); + let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); + std::fs::create_dir_all(&model_dir).expect("create model dir"); + + for file in TTS_EXPECTED_FILES { + std::fs::write(model_dir.join(file), b"test").expect("write expected file"); + } + std::fs::write(model_dir.join(MANIFEST_FILENAME), TTS_MODEL_VERSION).expect("manifest"); + + assert!(slot.is_ready(temp.path())); + + std::fs::remove_file(model_dir.join(TTS_LICENSE_FILE_NAME)).expect("remove sidecar"); + assert!(!slot.is_ready(temp.path())); +} + +#[test] +fn interrupted_install_restores_previous_model_directory() { + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); + let backup_dir = temp.path().join("pocket-tts.old"); + std::fs::create_dir_all(&backup_dir).expect("create backup"); + std::fs::write(backup_dir.join("sentinel"), b"january").expect("write sentinel"); + + slot.recover_interrupted_install(temp.path()); + + assert_eq!( + std::fs::read(temp.path().join(TTS_MODEL_DIR_NAME).join("sentinel")) + .expect("restored sentinel"), + b"january" + ); + assert!(!backup_dir.exists()); +} + +#[test] +fn interrupted_install_replaces_incomplete_destination_with_backup() { + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); + let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); + let backup_dir = temp.path().join("pocket-tts.old"); + std::fs::create_dir_all(&model_dir).expect("create incomplete destination"); + std::fs::write(model_dir.join("incomplete"), b"april").expect("write incomplete file"); + std::fs::create_dir_all(&backup_dir).expect("create backup"); + std::fs::write(backup_dir.join("sentinel"), b"january").expect("write sentinel"); + + slot.recover_interrupted_install(temp.path()); + + assert_eq!( + std::fs::read(model_dir.join("sentinel")).expect("restored sentinel"), + b"january" + ); + assert!(!model_dir.join("incomplete").exists()); + assert!(!backup_dir.exists()); +} diff --git a/desktop/src-tauri/src/huddle/pocket.rs b/desktop/src-tauri/src/huddle/pocket.rs index ee1faf928a..8e19781eeb 100644 --- a/desktop/src-tauri/src/huddle/pocket.rs +++ b/desktop/src-tauri/src/huddle/pocket.rs @@ -1,654 +1,3 @@ -//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. -//! -//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). -//! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". -//! -//! ## Attribution -//! -//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. -//! arXiv:2509.06926. Original repository: . -//! Licensed CC-BY-4.0. -//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. -//! - **ONNX export**: KevinAHM — -//! . CC-BY-4.0. -//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — -//! . -//! Repackages KevinAHM's export with the file layout sherpa-onnx's -//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. -//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary -//! (f, conversation)" preset from the Kyutai TTS demo -//! (), which maps to `vctk/p333_023_enhanced.wav` -//! in . CC-BY-4.0, base recording -//! from the VCTK corpus, enhanced by ai-coustics. -//! -//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` -//! sidecar written by `huddle::models` during install for the canonical -//! CC-BY-4.0 §3(a)(1) attribution block. -//! -//! ## Engine-module contract (see `huddle::tts`) -//! -//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this -//! contract is what lets the TTS pipeline stay engine-agnostic: -//! -//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. -//! - `DEFAULT_VOICE: &str` — default voice name (without extension). -//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. -//! - `load_text_to_speech(model_dir)` → `Result` -//! - `load_voice_style(path)` → `Result` -//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` -//! → `Result, String>` -//! -//! `lang` and `steps` are accepted for API compatibility with the previous -//! Kokoro engine but are unused — Pocket TTS does its own language ID from -//! the input text and is not a diffusion model (consistency LM, one step). -//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only -//! read by some model families (vits), never by the Pocket impl -//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts -//! has no speed parameter either. +//! Compatibility re-export for the shared Pocket TTS engine. -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -// ── Engine-module contract: public consts ───────────────────────────────────── - -/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, -/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. -pub const SAMPLE_RATE: u32 = 24_000; - -/// Name (without extension) of the bundled reference voice. The model directory -/// is expected to contain `.` after install. -pub const DEFAULT_VOICE: &str = "reference_sample"; - -/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the -/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, -/// not a precomputed embedding, so the extension change is honest. -pub const VOICE_FILE_EXT: &str = "wav"; - -// ── Tuning ──────────────────────────────────────────────────────────────────── - -/// Single-threaded ONNX execution for predictable CPU contention with the STT -/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark -/// argues for it. -const TTS_NUM_THREADS: i32 = 1; - -/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. -/// We bind to one voice per pipeline today, but the upstream example uses 16 -/// and the cost is negligible — keep room for future multi-voice support. -const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; - -/// Pocket TTS is a consistency-based LM. Generation quality saturates at one -/// denoising step — the upstream `GenerationConfig` default of 5 multiplies -/// synthesis time by ~5× with no audible benefit on this model. -const SYNTH_NUM_STEPS: i32 = 1; - -/// Leave the generated audio's silences untouched (1.0 is the identity). -/// -/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding -/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. -const SYNTH_SILENCE_SCALE: f32 = 1.0; - -/// sherpa-onnx upstream default for `max_frames` (LM steps), in -/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the -/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; -/// production code path never raises (or even reads) this value — we just -/// leave sherpa-onnx's own default in place by not setting the override. -#[cfg(test)] -const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; - -/// Tight `max_frames` we ask for on short, padded prompts to bound the -/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — -/// roomy for any one-to-four-word utterance the user is likely to elicit -/// while still well short of the 40 s upstream default. Chosen with slack so -/// we never *truncate* a legitimate short reply. -const SHORT_PROMPT_MAX_FRAMES: i32 = 100; - -/// Word-count threshold (inclusive) below which we pad the prompt with -/// leading spaces and cap `max_frames` tighter than the upstream default. -/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. -const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; - -/// Number of leading spaces prepended to short prompts. The upstream Python -/// uses exactly 8 — keep parity rather than tuning blindly. -/// -/// This is upstream's *only* mitigation for the FlowLM cold-start smear on -/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive -/// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. -const SHORT_PROMPT_PAD_SPACES: usize = 8; - -/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. -#[cfg(test)] -const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; - -// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── - -const FILE_LM_MAIN: &str = "lm_main.onnx"; -const FILE_LM_FLOW: &str = "lm_flow.onnx"; -const FILE_ENCODER: &str = "encoder.onnx"; -const FILE_DECODER: &str = "decoder.onnx"; -const FILE_TEXT_COND: &str = "text_conditioner.onnx"; -const FILE_VOCAB: &str = "vocab.json"; -const FILE_TOKEN_SCORES: &str = "token_scores.json"; - -// ── Voice style ─────────────────────────────────────────────────────────────── - -/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. -/// -/// Pocket TTS takes a reference waveform per generation call (not a -/// precomputed style embedding), so we keep the samples in memory and clone -/// the small `Vec` into each `GenerationConfig` rather than re-reading the -/// WAV from disk on every sentence. -#[derive(Debug, Clone)] -pub struct VoiceStyle { - samples: Vec, - sample_rate: i32, -} - -/// Load a reference voice WAV from disk. -/// -/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS -/// resamples internally using `reference_sample_rate`. The bundled -/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. -pub fn load_voice_style(path: &Path) -> Result { - let path_str = path - .to_str() - .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; - let wave = Wave::read(path_str) - .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; - let samples = wave.samples().to_vec(); - if samples.is_empty() { - return Err(format!("voice WAV is empty: {}", path.display())); - } - Ok(VoiceStyle { - samples, - sample_rate: wave.sample_rate(), - }) -} - -// ── Engine ──────────────────────────────────────────────────────────────────── - -/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` -/// call). Owned by the TTS worker thread for the lifetime of a huddle session. -/// -/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the -/// pipeline only needs to move the engine into the worker thread and call -/// `synth_chunk` on it, never to print it. -pub struct PocketTts { - inner: OfflineTts, -} - -/// Build the Pocket TTS engine from the model directory installed by -/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is -/// missing — readiness is normally enforced by `is_tts_ready` upstream, but -/// the check is repeated here so a manually-modified model dir produces a -/// clear error string instead of an opaque sherpa-onnx `None`. -pub fn load_text_to_speech(model_dir: &str) -> Result { - let dir = PathBuf::from(model_dir); - for name in [ - FILE_LM_MAIN, - FILE_LM_FLOW, - FILE_ENCODER, - FILE_DECODER, - FILE_TEXT_COND, - FILE_VOCAB, - FILE_TOKEN_SCORES, - ] { - let p = dir.join(name); - if !p.is_file() { - return Err(format!("missing Pocket TTS file: {}", p.display())); - } - } - - let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; - - // Build the config by mutating defaults — mirrors `stt.rs` and stays - // resilient if sherpa-onnx adds unrelated model-family fields. - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); - cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); - cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); - cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); - cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); - cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); - cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); - cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; - cfg.model.num_threads = TTS_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every synthesized chunk. - cfg.model.debug = false; - - let inner = OfflineTts::create(&cfg) - .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; - Ok(PocketTts { inner }) -} - -// ── Prompt preparation ──────────────────────────────────────────────────────── - -/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the -/// per-call generation overrides derived from the original text. -/// -/// `None` for either override means "leave sherpa-onnx's documented default -/// in place". The pipeline only sets `max_frames` (and only for short -/// padded inputs) so it can bound the original "monster breathing" runaway -/// without disturbing the rest of the LM sampling envelope. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PreparedPrompt { - /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, - /// punctuation-terminated, and (for short inputs) left-padded with - /// spaces — upstream's mitigation for the FlowLM cold-start smear. - pub text: String, - /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to - /// keep the upstream default of 500 LM steps. We only override on short - /// padded prompts where we have a tight expectation on output length. - pub max_frames: Option, -} - -/// Mirror of the *text-preparation* half of upstream -/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ -/// Pocket TTS impl does not run these preparation steps, so short / -/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway -/// generation when the EOS logit never crosses its threshold. We replicate -/// the upstream Python recipe here: -/// -/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but -/// cheap to re-check after sentence splitting). -/// 2. Capitalize the first letter. -/// 3. Append `.` if the text doesn't end in punctuation. -/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces -/// (upstream's cold-start mitigation — see the constant's docstring) and -/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run -/// away if EOS still doesn't fire. -/// -/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 -/// is what we want. An earlier version set it to 1 on long inputs, which -/// clipped the leading audio of multi-clause sentences ("first 'yep' is -/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` -/// lock this in. -/// -/// Returns `None` only if the input is empty after trimming — caller should -/// skip synthesis in that case. -pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { - let trimmed = input.trim(); - if trimmed.is_empty() { - return None; - } - - // Collapse stray double-spaces / embedded newlines that may slip past - // `preprocess_for_tts` when sentences are spliced back together. - let mut cleaned = String::with_capacity(trimmed.len()); - let mut last_was_space = false; - for ch in trimmed.chars() { - let is_ws = ch.is_whitespace(); - if is_ws { - if !last_was_space { - cleaned.push(' '); - } - last_was_space = true; - } else { - cleaned.push(ch); - last_was_space = false; - } - } - - // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). - let first = cleaned.chars().next().expect("cleaned non-empty above"); - if first.is_lowercase() { - let upper: String = first.to_uppercase().collect(); - let mut iter = cleaned.chars(); - iter.next(); - cleaned = upper + iter.as_str(); - } - - // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. - // The upstream Python only checks `isalnum` → period, but for our agent - // text we already may end in `!` `?` `.` etc. — treat any of those as OK. - let last = cleaned - .chars() - .next_back() - .expect("cleaned non-empty above"); - if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { - cleaned.push('.'); - } - - // Word count of the *cleaned but not padded* text — padding is whitespace - // only and would just lie to the threshold check below. - let word_count = cleaned.split_whitespace().count(); - - let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { - let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); - for _ in 0..SHORT_PROMPT_PAD_SPACES { - padded.push(' '); - } - padded.push_str(&cleaned); - (padded, Some(SHORT_PROMPT_MAX_FRAMES)) - } else { - // For everything ≥5 words, fall back to upstream defaults. Overriding - // these is what caused the "first 'yep' is static" regression — the - // upstream LM has been tuned for `frames_after_eos = 3` and - // `max_frames = 500`, and there's no clear win in second-guessing. - (cleaned, None) - }; - - Some(PreparedPrompt { - text: final_text, - max_frames, - }) -} - -/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. -/// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. -fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { - prepared.max_frames.map(|mf| { - let mut h: HashMap = HashMap::with_capacity(1); - h.insert("max_frames".to_string(), serde_json::Value::from(mf)); - h - }) -} - -impl PocketTts { - /// Synthesise `text` with the given reference voice. - /// - /// `_lang` and `_steps` are accepted for API compatibility with the - /// previous Kokoro engine. Pocket TTS infers language from the input text - /// directly and is a one-step consistency model. Returns an empty buffer - /// for whitespace-only input. - pub fn synth_chunk( - &self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _steps: usize, - ) -> Result, String> { - // Mirror upstream pocket-tts prompt prep — without this short or - // unpunctuated inputs can cause the LM's EOS logit to never trip, - // producing up to 40 s of "monster breathing" garbage on the first - // utterance. See `prepare_pocket_prompt` for the full recipe. - let prepared = match prepare_pocket_prompt(text) { - Some(p) => p, - None => return Ok(Vec::new()), - }; - - // Per-call generation hints sherpa-onnx forwards to - // `offline-tts-pocket-impl.h`. We only override `max_frames`, and - // only for short padded prompts where we have a tight expectation - // on output length — that bounds the original runaway without - // disturbing the rest of the LM sampling envelope. See - // `prepare_pocket_prompt` docs for the regression history. - let extra = build_generation_extra(&prepared); - - let cfg = GenerationConfig { - num_steps: SYNTH_NUM_STEPS, - silence_scale: SYNTH_SILENCE_SCALE, - reference_audio: Some(style.samples.clone()), - reference_sample_rate: style.sample_rate, - extra, - // `speed` stays at its default: the Pocket impl never reads it - // (see the engine-contract note in the module docs). - ..Default::default() - }; - - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. - let audio = self - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| { - format!( - "Pocket TTS synthesis failed for text ({} chars)", - prepared.text.len() - ) - })?; - - let sample_rate = audio.sample_rate(); - if sample_rate != SAMPLE_RATE as i32 { - eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ - (expected {SAMPLE_RATE}Hz); playback speed may be wrong" - ); - } - - Ok(audio.samples().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── prepare_pocket_prompt ──────────────────────────────────────────────── - - #[test] - fn prepare_prompt_returns_none_for_empty_input() { - assert!(prepare_pocket_prompt("").is_none()); - assert!(prepare_pocket_prompt(" ").is_none()); - assert!(prepare_pocket_prompt("\n\t ").is_none()); - } - - /// Helper: the exact leading sequence prepended to every short prompt — - /// 8 spaces of padding (upstream's cold-start mitigation). - /// Centralising this keeps the assertions readable. - fn short_prefix() -> String { - " ".repeat(SHORT_PROMPT_PAD_SPACES) - } - - #[test] - fn prepare_prompt_pads_and_capitalizes_one_word() { - // The "yep" case Tyler hit in production — bare lowercase one-word - // utterance with no punctuation. Must be padded with the short-prompt - // space pad, capitalized, terminated, with a tight `max_frames` cap - // to bound runaway gen. - let out = prepare_pocket_prompt("yep").expect("non-empty"); - assert_eq!(out.text, format!("{}Yep.", short_prefix())); - assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - const { - assert!( - SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, - "short cap must be tighter than the upstream default" - ); - } - } - - #[test] - fn prepare_prompt_preserves_existing_punctuation() { - let out = prepare_pocket_prompt("yes!").expect("non-empty"); - assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept - let out = prepare_pocket_prompt("really?").expect("non-empty"); - assert_eq!(out.text, format!("{}Really?", short_prefix())); - } - - #[test] - fn prepare_prompt_threshold_is_inclusive_at_four_words() { - // 4 words = short (padded + tight max_frames); 5 words = long - // (no padding, no overrides — upstream defaults stand). - let four = prepare_pocket_prompt("one two three four").expect("non-empty"); - assert_eq!( - four.text, - format!("{}One two three four.", short_prefix()), - "four-word input should get exactly the space pad" - ); - assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - - let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); - assert!( - !five.text.starts_with(' '), - "five-word input should NOT be padded" - ); - assert_eq!( - five.max_frames, None, - "long inputs must leave sherpa-onnx's max_frames default in place" - ); - } - - #[test] - fn prepare_prompt_does_not_pad_long_text() { - let long = "This is a longer sentence that the model should handle just fine."; - let out = prepare_pocket_prompt(long).expect("non-empty"); - assert!(!out.text.starts_with(' ')); - assert_eq!(out.max_frames, None); - assert!(out.text.ends_with('.')); - } - - #[test] - fn prepare_prompt_collapses_whitespace() { - let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); - // 3 words → short → padded. Interior whitespace collapsed. - assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); - } - - #[test] - fn prepare_prompt_does_not_double_capitalize_already_uppercase() { - let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); - assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); - } - - #[test] - fn prepare_prompt_handles_non_ascii_first_letter() { - // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce - // mojibake. - let out = prepare_pocket_prompt("дa").expect("non-empty"); - assert!(out.text.contains("Дa.")); - } - - /// REGRESSION GUARD: short prompts must receive *only* whitespace - /// padding — no sacrificial text. A previous revision prepended a - /// `". . "` cold-start absorber and trimmed the rendered audio back out - /// with an amplitude threshold that could eat soft word onsets. If - /// non-whitespace ever reappears in the pad, the synth output will - /// contain audio for text the user never wrote. - #[test] - fn prepare_prompt_pad_is_whitespace_only() { - let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); - let pad_len = out.text.len() - "I'm happy.".len(); - assert!( - out.text[..pad_len].chars().all(|c| c == ' '), - "short-prompt pad must be spaces only, got {:?}", - &out.text[..pad_len] - ); - assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); - } - - // ── build_generation_extra ─────────────────────────────────────────────── - // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). - - #[test] - fn build_extra_short_prompt_sets_only_max_frames() { - let prepared = prepare_pocket_prompt("yep").expect("non-empty"); - let extra = build_generation_extra(&prepared).expect("short prompts get extra"); - // Exactly one key — `max_frames` — and nothing else. - assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); - assert_eq!( - extra.get("max_frames"), - Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) - ); - assert!( - !extra.contains_key("frames_after_eos"), - "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" - ); - } - - #[test] - fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. - let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); - assert_eq!( - build_generation_extra(&prepared), - None, - "long prompts must not override any LM knob" - ); - } - - #[test] - fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { - // Sweep a range of prompt lengths and assert the `extra` map (when - // present) never carries a `frames_after_eos` override that's lower - // than the upstream sherpa-onnx default. Implemented as a structural - // check — we just never set the key — but worth a property test in - // case someone reintroduces the override in the future. - let prompts: &[&str] = &[ - "hi", - "hi there", - "yes please", - "one two three four", - "one two three four five", - "a slightly longer reply, hopefully fine", - "This is a multi-clause sentence. It has two parts.", - "really really really really really long prompt with lots of words just to be sure", - ]; - for &p in prompts { - let prepared = prepare_pocket_prompt(p).expect("non-empty"); - if let Some(extra) = build_generation_extra(&prepared) { - if let Some(v) = extra.get("frames_after_eos") { - let n = v.as_i64().expect("frames_after_eos should be int"); - assert!( - n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, - "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" - ); - } - } - } - } - - #[test] - fn short_prompt_max_frames_is_below_upstream_default() { - // Sanity: the override only ever *lowers* the cap, never raises it. - const { - assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); - } - // …and is still large enough for a one-to-four-word reply. At Mimi's - // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. - const { - assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); - } - } -} +pub use buzz_voice_pkg::pocket::*; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 63a435cd8e..0bcb73c62c 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -74,8 +74,7 @@ const SYNTH_STEPS: usize = 1; /// Applied only at the *end* of each synthesised sentence to eliminate the /// click that would otherwise occur when a non-zero waveform terminates /// abruptly. **No fade-in is applied** — see `apply_fade_out` for the -/// rationale and `examples/pocket_onset_probe.rs` for the measurement that -/// motivated removing the leading fade. +/// rationale and measurements that motivated removing the leading fade. const FADE_OUT_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; /// Length of the zero-sample cushion prepended before each synthesized @@ -101,13 +100,10 @@ const SENTENCE_LEAD_IN_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.020) as usize; /// names chunk stitching as the reliability lever). Our previous /// sentence-per-call path created ~2–4× more seams than upstream. /// -/// We don't ship the SentencePiece tokenizer, so 50 tokens is approximated -/// with a character budget. The bundled 4k-entry vocab averages ~4 chars per -/// token, but usage-weighted English text leans on short common tokens, so -/// the effective ratio is ~2–4 chars/token and 200 chars ≈ 60–100 tokens — -/// modestly above upstream's 50, deliberately: erring large means fewer -/// seams, and even ~100 tokens is far below the model's 500-LM-step (~40 s) -/// ceiling. Do not shrink this budget to chase an exact 50-token match. +/// Chunk grouping happens at the pipeline boundary, before the April engine's +/// SentencePiece tokenizer runs, so this character budget performs only the +/// coarse sentence packing. The loaded engine then refines every result at +/// the bundle's exact 50-token boundary before the worker synthesis loop. const MAX_CHUNK_CHARS: usize = 200; /// Silence inserted between sentences by the TTS pipeline (seconds). @@ -502,7 +498,22 @@ fn tts_worker( .into_iter() .filter(|s| !s.trim().is_empty()) .collect(); - let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); + let grouped_chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); + let mut chunks = Vec::new(); + let mut split_failed = false; + for chunk in grouped_chunks { + match engine.split_text_into_chunks(&chunk) { + Ok(model_chunks) => chunks.extend(model_chunks), + Err(e) => { + eprintln!("buzz-desktop: TTS chunking failed: {e}"); + split_failed = true; + break; + } + } + } + if split_failed { + continue; + } for chunk in &chunks { if handle_cancel_or_shutdown( @@ -647,8 +658,8 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> { /// Hard-clamp samples to ±1.0 full scale. /// /// No gain is applied: Pocket TTS already emits speech-level audio -/// (peaks 0.4–0.97, RMS ≈ −20 dBFS across varied sentences — measured by -/// `examples/pocket_clip_probe`), matching the kyutai reference pipeline, +/// (peaks 0.4–0.97, RMS ≈ −20 dBFS across varied sentences), matching the +/// kyutai reference pipeline, /// which applies no output scaling. Two earlier gain stages were both /// regressions against that baseline: per-sentence peak normalization caused /// level pumping between sentences, and the fixed 9.3× gain that replaced it @@ -670,8 +681,8 @@ fn clamp_to_full_scale(samples: Vec) -> Vec { /// An earlier revision (pre 2026-05) symmetrically faded *in* over the same /// 8 ms window. That swallowed the leading consonant attack on every /// sentence — Pocket TTS produces real audio energy inside the first -/// millisecond (RMS ≈ 0.02, peak ≈ 0.03 measured across four prompts in -/// `examples/pocket_onset_probe.rs`), and a linear 0→1 ramp over 192 samples +/// millisecond (RMS ≈ 0.02, peak ≈ 0.03 across four measured prompts), and a +/// linear 0→1 ramp over 192 samples /// scales those onset samples by ≤50 % for the first ~4 ms. The result was /// the "first little sound or two is missing" regression heard on /// 2026-05-18. @@ -697,7 +708,6 @@ fn apply_fade_out(samples: &mut [f32]) { /// the synthesized buffer can begin with speech within the first millisecond, /// so the playback layer must provide the device/mixer cushion. /// To keep the audible gap unchanged, the trailing silence after this chunk is -/// shortened by the same amount (`silence_buf_len - SENTENCE_LEAD_IN_SAMPLES`): /// sentence N contributes 80 ms of post-speech silence and sentence N+1 /// contributes the remaining 20 ms of pre-speech cushion. /// @@ -734,9 +744,9 @@ fn build_sentence_append_buffer( /// single-sentence cost. Subsequent sentences pack greedily: a sentence /// joins the current chunk while the combined length stays within /// `max_chars`; otherwise it starts a new chunk. A single sentence longer -/// than `max_chars` becomes its own chunk unsplit — Pocket TTS handles long -/// single sentences fine (the ceiling is the 500-LM-step default), it's the -/// *seams* we're minimizing. +/// than `max_chars` becomes its own chunk here, then the Pocket engine splits +/// it at the April bundle's exact token limit before synthesis. Keeping that +/// second split in the worker preserves per-chunk fades and cancel checks. /// /// Sentences within a chunk are joined with a single space; sentence-ending /// punctuation is preserved by `split_sentences`, so the model sees natural diff --git a/desktop/src-tauri/src/huddle/tts_tests.rs b/desktop/src-tauri/src/huddle/tts_tests.rs index 7887f8bbdb..904749bf08 100644 --- a/desktop/src-tauri/src/huddle/tts_tests.rs +++ b/desktop/src-tauri/src/huddle/tts_tests.rs @@ -950,8 +950,8 @@ fn chunk_grouping_packs_up_to_budget_then_spills() { } /// A single sentence longer than the budget is passed through unsplit — -/// long single sentences are fine (the LM cap bounds runaway); only seams -/// are being minimized. +/// exact token splitting happens afterward through the loaded Pocket engine +/// so every result remains a normal playback/cancellation unit. #[test] fn chunk_grouping_oversized_sentence_passes_through() { let long = "word ".repeat(60).trim_end().to_string() + "."; diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4dd1142801..ae26fc6ba8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -84,6 +84,9 @@ run_unit_tests() { run_test_step "buzz-auth unit tests" \ cargo test -p buzz-auth --lib -- --nocapture + run_test_step "buzz-voice tests" \ + cargo test -p buzz-voice --lib -- --nocapture + # buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator # invariant (exactly the consolidated 0001; cutover/backfill stays an operator # script, not startup state) and the tenant-scoping lints. The Postgres-backed