Adapt to the new seq_len cli-flag of gemma.cpp

Author: ufownl

README.md

@@ -171,6 +171,7 @@ Available options and default values:
 
 ```lua
 {
+  seq_len = 2048,  -- Sequence length, capped by max context window of model.
   max_generated_tokens = 2048,  -- Maximum number of tokens to generate.
   prefill_tbatch = 256,  -- Prefill: max tokens per batch.
   decode_qbatch = 16,  -- Decode: max queries per batch.

demo/src/init.lua

@@ -9,6 +9,7 @@ function config()
       weights = "4b-it-sfp.sbs"
     },
     session = {
+      seq_len = 8192,
       temperature = 0.4,
       top_k = 5
     },

demo/src/init_fn.lua

@@ -9,6 +9,7 @@ function config()
       weights = "12b-it-sfp.sbs"
     },
     session = {
+      seq_len = 8192,
       temperature = 0.4,
       top_k = 5
     },

demo/src/init_huggingface.lua

@@ -10,6 +10,7 @@ function config()
       weights = "4b-it-sfp.sbs"
     },
     session = {
+      seq_len = 8192,
       temperature = 0.4,
       top_k = 5
     },

demo/src/init_kaggle.lua

@@ -6,6 +6,7 @@ function config()
       weights = "4b-it-sfp.sbs"
     },
     session = {
+      seq_len = 8192,
       prefill_tbatch = 64,
       temperature = 0.4,
       top_k = 5

demo/src/init_vlm.lua

@@ -11,6 +11,7 @@ function config()
       weights = "4b-it-sfp.sbs"
     },
     session = {
+      seq_len = 8192,
       temperature = 0.4,
       top_k = 5
     },

examples/normal_mode.lua

@@ -5,6 +5,10 @@ if args.help then
     "resty normal_mode.lua [options]"
   )
   print("  --image: Path of image file (PPM format: P6, binary).")
+  print("  --seq_len: Sequence length, capped by max context window of model. (default: 4096)")
+  print("  --max_generated_tokens: Maximum number of tokens to generate. (default: 2048)")
+  print("  --temperature: Temperature for top-K. (default: 1.0)")
+  print("  --top_k: Number of top-K tokens to sample from. (default: 5)")
   print("  --kv_cache: Path of KV cache file.")
   print("  --stats: Print statistics at end of turn.")
   return
@@ -29,7 +33,12 @@ if args.image then
 end
 
 -- Create a chat session
-local session, err = gemma:session({top_k = 5})
+local session, err = gemma:session({
+  seq_len = tonumber(args.seq_len) or 4096,
+  max_generated_tokens = tonumber(args.max_generated_tokens),
+  temperature = tonumber(args.temperature),
+  top_k = tonumber(args.top_k) or 5
+})
 if not session then
   error("Opoos! "..err)
 end

examples/stream_mode.lua

@@ -5,6 +5,10 @@ if args.help then
     "resty stream_mode.lua [options]"
   )
   print("  --image: Path of image file (PPM format: P6, binary).")
+  print("  --seq_len: Sequence length, capped by max context window of model. (default: 4096)")
+  print("  --max_generated_tokens: Maximum number of tokens to generate. (default: 2048)")
+  print("  --temperature: Temperature for top-K. (default: 1.0)")
+  print("  --top_k: Number of top-K tokens to sample from. (default: 5)")
   print("  --kv_cache: Path of KV cache file.")
   print("  --stats: Print statistics at end of turn.")
   return
@@ -29,7 +33,12 @@ if args.image then
 end
 
 -- Create a chat session
-local session, err = gemma:session({top_k = 5})
+local session, err = gemma:session({
+  seq_len = tonumber(args.seq_len) or 4096,
+  max_generated_tokens = tonumber(args.max_generated_tokens),
+  temperature = tonumber(args.temperature),
+  top_k = tonumber(args.top_k) or 5
+})
 if not session then
   error("Opoos! "..err)
 end

src/session.cpp

@@ -383,6 +383,7 @@ int session::create(lua_State* L) {
   auto nargs = lua_gettop(L);
   auto inst = instance::check(L, 1);
   constexpr const char* available_options[] = {
+    "--seq_len",
     "--max_generated_tokens",
     "--prefill_tbatch",
     "--decode_qbatch",

tools/dump_prompt.lua

@@ -20,6 +20,7 @@ if args.help then
   print("  --tokenizer: Path of tokenizer model file. (default: tokenizer.spm)")
   print("  --weights: Path of model weights file. (default: 4b-it-sfp.sbs)")
   print("  --map: Enable memory-mapping? -1 = auto, 0 = no, 1 = yes. (default: -1)")
+  print("  --seq_len: Sequence length, capped by max context window of model. (default: 4096)")
   print("  --max_generated_tokens: Maximum number of tokens to generate. (default: 2048)")
   print("  --prefill_tbatch: Maximum batch size during prefill phase (default: 256)")
   print("  --temperature: Temperature for top-K. (default: 1.0)")
@@ -43,14 +44,14 @@ end
 
 -- Config global scheduler
 local ok, err = require("cgemma").scheduler.config({
-  num_threads = args.num_threads,
-  pin = args.pin,
-  skip_packages = args.skip_packages,
-  max_packages = args.max_packages,
-  skip_clusters = args.skip_clusters,
-  max_clusters = args.max_clusters,
-  skip_lps = args.skip_lps,
-  max_lps = args.max_lps
+  num_threads = tonumber(args.num_threads),
+  pin = tonumber(args.pin),
+  skip_packages = tonumber(args.skip_packages),
+  max_packages = tonumber(args.max_packages),
+  skip_clusters = tonumber(args.skip_clusters),
+  max_clusters = tonumber(args.max_clusters),
+  skip_lps = tonumber(args.skip_lps),
+  max_lps = tonumber(args.max_lps)
 })
 if not ok then
   error("Opoos! "..err)
@@ -78,10 +79,11 @@ end
 
 -- Create a session
 local session, err = gemma:session({
-  max_generated_tokens = args.max_generated_tokens,
-  prefill_tbatch = args.prefill_tbatch,
-  temperature = args.temperature,
-  top_k = args.top_k,
+  seq_len = tonumber(args.seq_len) or 4096,
+  max_generated_tokens = tonumber(args.max_generated_tokens),
+  prefill_tbatch = tonumber(args.prefill_tbatch),
+  temperature = tonumber(args.temperature),
+  top_k = tonumber(args.top_k),
   no_wrapping = true
 })
 if not session then