From 0b668e408927f5ecc6a18c63dfdceefc24727723 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 4 Jun 2026 21:19:45 +0000 Subject: [PATCH] feat: per-request metadata to disable thinking/reasoning Let users attach a metadata object to every LLM request (the OpenAI "metadata" field), passed verbatim to the backend. Backends such as LocalAI use it for per-request flags, e.g. {"enable_thinking": "false"} to disable a reasoning model's thinking. - Config.Metadata (global) and AgentTypeConfig.Metadata (per-agent). - Main session sends Config.Metadata; sub-agents merge global + per-agent via mergeMetadata (per key: agent wins, global-only keys inherited). - e2e tests assert metadata reaches the wire; README documents both. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 9 ++ chat/agentmodel_test.go | 42 ++++++++- chat/session.go | 33 +++++++- chat/session_metadata_test.go | 155 ++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- types/config.go | 19 +++-- 7 files changed, 252 insertions(+), 12 deletions(-) create mode 100644 chat/session_metadata_test.go diff --git a/README.md b/README.md index 239b0f6..b39cde7 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,12 @@ base_url: https://api.openai.com/v1 prompt: | You are a calm, helpful terminal assistant... +# Optional: per-request metadata sent verbatim on every LLM request (the OpenAI +# "metadata" object). Backends such as LocalAI use it for per-request flags — +# e.g. disable a reasoning model's thinking: +metadata: + enable_thinking: "false" + # Optional: agent behavior agent_options: iterations: 10 @@ -242,6 +248,9 @@ agents: description: investigates a self-contained subtask system_prompt: You are a focused research sub-agent. tools: [bash] + # Per-agent metadata overlays the global metadata above (per key): + metadata: + enable_thinking: "true" # Optional: external MCP servers mcp_servers: diff --git a/chat/agentmodel_test.go b/chat/agentmodel_test.go index 5da8d0f..5c35487 100644 --- a/chat/agentmodel_test.go +++ b/chat/agentmodel_test.go @@ -1,6 +1,46 @@ package chat -import "testing" +import ( + "reflect" + "testing" +) + +func TestMergeMetadata(t *testing.T) { + cases := []struct { + name string + global map[string]string + override map[string]string + want map[string]string + }{ + {"both empty -> nil", nil, nil, nil}, + {"global only", map[string]string{"enable_thinking": "false"}, nil, map[string]string{"enable_thinking": "false"}}, + {"override only", nil, map[string]string{"enable_thinking": "true"}, map[string]string{"enable_thinking": "true"}}, + { + "override wins per key, global-only inherited", + map[string]string{"enable_thinking": "false", "tier": "low"}, + map[string]string{"enable_thinking": "true"}, + map[string]string{"enable_thinking": "true", "tier": "low"}, + }, + } + for _, c := range cases { + got := mergeMetadata(c.global, c.override) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("%s: mergeMetadata(%v, %v) = %v, want %v", c.name, c.global, c.override, got, c.want) + } + } +} + +func TestMergeMetadataDoesNotMutateInputs(t *testing.T) { + global := map[string]string{"enable_thinking": "false"} + override := map[string]string{"enable_thinking": "true"} + _ = mergeMetadata(global, override) + if global["enable_thinking"] != "false" { + t.Errorf("global was mutated: %v", global) + } + if override["enable_thinking"] != "true" { + t.Errorf("override was mutated: %v", override) + } +} func TestResolveAgentModel(t *testing.T) { main := "qwen-main" diff --git a/chat/session.go b/chat/session.go index 4bcd6d1..415ffb2 100644 --- a/chat/session.go +++ b/chat/session.go @@ -59,6 +59,7 @@ type Session struct { llmModel string apiKey string baseURL string + metadata map[string]string // global per-request metadata; merged with per-agent overrides configurator *manage.Configurator reloadMu sync.Mutex @@ -87,6 +88,23 @@ func resolveAgentModel(requested, main string, configured map[string]bool) strin return main } +// mergeMetadata overlays per-agent metadata on top of the global metadata, +// returning a new map (or nil when both are empty). Per-agent keys win; keys +// present only in the global map are inherited. Inputs are never mutated. +func mergeMetadata(global, override map[string]string) map[string]string { + if len(global) == 0 && len(override) == 0 { + return nil + } + merged := make(map[string]string, len(global)+len(override)) + for k, v := range global { + merged[k] = v + } + for k, v := range override { + merged[k] = v + } + return merged +} + // agentModelSet collects the non-empty per-agent-type model overrides so the // agent LLM factory can tell a configured model apart from an invented one. func agentModelSet(defs []cogito.AgentDefinition) map[string]bool { @@ -110,6 +128,7 @@ func toCogitoDefinitions(cfgs []types.AgentTypeConfig) []cogito.AgentDefinition Tools: t.Tools, Model: t.Model, Temperature: t.Temperature, + Metadata: t.Metadata, Iterations: t.Iterations, MaxAttempts: t.MaxAttempts, MaxRetries: t.MaxRetries, @@ -130,7 +149,9 @@ func CommandTransport(cmd string, args []string, env ...string) mcp.Transport { // NewSession creates a new chat session func NewSession(ctx context.Context, cfg types.Config, callbacks Callbacks, transports ...mcp.Transport) (*Session, error) { - var llm cogito.LLM = clients.NewOpenAILLM(cfg.Model, cfg.APIKey, cfg.BaseURL) + var llm cogito.LLM = clients.NewOpenAILLMWithOptions(cfg.Model, cfg.APIKey, cfg.BaseURL, clients.OpenAIOptions{ + Metadata: cfg.Metadata, + }) // Session tracing: wrap the LLM so every call is appended to the transcript. // A recorder failure must never prevent the session from starting. @@ -177,6 +198,7 @@ func NewSession(ctx context.Context, cfg types.Config, callbacks Callbacks, tran llmModel: cfg.Model, apiKey: cfg.APIKey, baseURL: cfg.BaseURL, + metadata: cfg.Metadata, mcpClient: client, cfgClients: map[string]*mcp.ClientSession{}, cfgServers: map[string]types.MCPServer{}, @@ -453,7 +475,7 @@ func (s *Session) SendMessage(text string) (string, error) { cogito.EnableAgentSpawning, cogito.WithAgentManager(s.agentManager), cogito.WithAgentDefinitions(s.agentDefs...), - cogito.WithAgentLLMFactory(func(model string, temperature float32) cogito.LLM { + cogito.WithAgentLLMFactory(func(model string, temperature float32, metadata map[string]string) cogito.LLM { // The spawn_agent tool lets the LLM request a `model`, which it may // fill with a name the endpoint doesn't serve (e.g. "sonar") and // 404 the sub-agent. Honor a requested model only when wiz actually @@ -465,7 +487,12 @@ func (s *Session) SendMessage(text string) (string, error) { xlog.Warn("sub-agent requested an unserved model; using the main model", "requested", model, "model", chosen) } - return clients.NewOpenAILLMWithOptions(chosen, s.apiKey, s.baseURL, clients.OpenAIOptions{Temperature: temperature}) + // metadata is this agent type's override; overlay it on the global + // session metadata (per-key: agent wins, global-only keys inherited). + return clients.NewOpenAILLMWithOptions(chosen, s.apiKey, s.baseURL, clients.OpenAIOptions{ + Temperature: temperature, + Metadata: mergeMetadata(s.metadata, metadata), + }) }), cogito.WithAgentSpawnCallback(func(a *cogito.AgentState) { s.emitAgentEvent(a) diff --git a/chat/session_metadata_test.go b/chat/session_metadata_test.go new file mode 100644 index 0000000..a195896 --- /dev/null +++ b/chat/session_metadata_test.go @@ -0,0 +1,155 @@ +package chat_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/mudler/nib/chat" + "github.com/mudler/nib/types" + "github.com/mudler/xlog" +) + +// metadataCapturingOpenAI is a minimal OpenAI-compatible endpoint that records +// the "metadata" object of every request it receives and always replies with a +// plain stop message so each turn ends in a single call. It handles both the +// streaming and non-streaming paths. +func metadataCapturingOpenAI(record func(map[string]string)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Stream bool `json:"stream"` + Metadata map[string]string `json:"metadata"` + } + body, _ := readAll(r) + _ = json.Unmarshal(body, &req) + record(req.Metadata) + + if !req.Stream { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "fake", "object": "chat.completion", "model": "fake", + "choices": []any{map[string]any{ + "index": 0, "message": map[string]any{"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + }}, + }) + return + } + + w.Header().Set("Content-Type", "text/event-stream") + fl, _ := w.(http.Flusher) + emit := func(delta map[string]any, finish string) { + choice := map[string]any{"index": 0, "delta": delta} + if finish != "" { + choice["finish_reason"] = finish + } + b, _ := json.Marshal(map[string]any{ + "id": "fake", "object": "chat.completion.chunk", "model": "fake", + "choices": []any{choice}, + }) + w.Write([]byte("data: ")) + w.Write(b) + w.Write([]byte("\n\n")) + if fl != nil { + fl.Flush() + } + } + emit(map[string]any{"content": "ok"}, "") + emit(map[string]any{}, "stop") + w.Write([]byte("data: [DONE]\n\n")) + if fl != nil { + fl.Flush() + } + } +} + +// TestSessionSendsConfiguredMetadata drives a real chat.Session against a fake +// LLM and proves the configured per-request metadata reaches the wire as the +// OpenAI "metadata" object on the main session's requests. +func TestSessionSendsConfiguredMetadata(t *testing.T) { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("error"), "")) + + var mu sync.Mutex + var last map[string]string + srv := httptest.NewServer(metadataCapturingOpenAI(func(m map[string]string) { + mu.Lock() + last = m + mu.Unlock() + })) + defer srv.Close() + + cfg := types.Config{ + Model: "fake-model", + APIKey: "fake-key", + BaseURL: srv.URL + "/v1", + LogLevel: "error", + ApprovalMode: "auto", + AgentOptions: types.AgentOptions{Iterations: 10, MaxAttempts: 3, MaxRetries: 3}, + Metadata: map[string]string{"enable_thinking": "false"}, + } + + session, err := chat.NewSession(context.Background(), cfg, chat.Callbacks{}) + if err != nil { + t.Fatalf("NewSession: %v", err) + } + defer session.Close() + + if _, err := session.SendMessage("hi"); err != nil { + t.Fatalf("SendMessage: %v", err) + } + + mu.Lock() + got := last + mu.Unlock() + if got["enable_thinking"] != "false" { + t.Fatalf("request metadata = %v, want enable_thinking=false", got) + } +} + +// TestSessionSendsNoMetadataByDefault proves that with no metadata configured, +// no "metadata" object is attached (the field stays omitted). +func TestSessionSendsNoMetadataByDefault(t *testing.T) { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("error"), "")) + + var mu sync.Mutex + var seen bool + var last map[string]string + srv := httptest.NewServer(metadataCapturingOpenAI(func(m map[string]string) { + mu.Lock() + seen = true + last = m + mu.Unlock() + })) + defer srv.Close() + + cfg := types.Config{ + Model: "fake-model", + APIKey: "fake-key", + BaseURL: srv.URL + "/v1", + LogLevel: "error", + ApprovalMode: "auto", + AgentOptions: types.AgentOptions{Iterations: 10, MaxAttempts: 3, MaxRetries: 3}, + } + + session, err := chat.NewSession(context.Background(), cfg, chat.Callbacks{}) + if err != nil { + t.Fatalf("NewSession: %v", err) + } + defer session.Close() + + if _, err := session.SendMessage("hi"); err != nil { + t.Fatalf("SendMessage: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if !seen { + t.Fatal("server received no request") + } + if len(last) != 0 { + t.Fatalf("expected no metadata, got %v", last) + } +} diff --git a/go.mod b/go.mod index a754c99..f472004 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/charmbracelet/glamour v1.0.0 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/modelcontextprotocol/go-sdk v1.0.0 - github.com/mudler/cogito v0.10.1-0.20260604082319-fe7fd5de11d1 + github.com/mudler/cogito v0.10.1-0.20260604212312-77024e447c46 github.com/mudler/xlog v0.0.1 github.com/sashabaranov/go-openai v1.41.2 golang.org/x/term v0.36.0 diff --git a/go.sum b/go.sum index 2c35042..d832a7b 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/modelcontextprotocol/go-sdk v1.0.0 h1:Z4MSjLi38bTgLrd/LjSmofqRqyBiVKR github.com/modelcontextprotocol/go-sdk v1.0.0/go.mod h1:nYtYQroQ2KQiM0/SbyEPUWQ6xs4B95gJjEalc9AQyOs= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mudler/cogito v0.10.1-0.20260604082319-fe7fd5de11d1 h1:2Xo02iDGVELS8fjlMM+A9z0Nbwp8fADd/PS+R+Ocin4= -github.com/mudler/cogito v0.10.1-0.20260604082319-fe7fd5de11d1/go.mod h1:6sfja3lcu2nWRzEc0wwqGNu/eCG3EWgij+8s7xyUeQ4= +github.com/mudler/cogito v0.10.1-0.20260604212312-77024e447c46 h1:lZGsJnVZSXE6J8BJq4frtmpFU+JYc4zBzP5YCULGZYU= +github.com/mudler/cogito v0.10.1-0.20260604212312-77024e447c46/go.mod h1:6sfja3lcu2nWRzEc0wwqGNu/eCG3EWgij+8s7xyUeQ4= github.com/mudler/xlog v0.0.1 h1:yR3/wszd3ZM6u1n96YITJZ4yUcDgqHSwvQmzUJa+8vg= github.com/mudler/xlog v0.0.1/go.mod h1:39f5vcd05Qd6GWKM8IjyHNQ7AmOx3ZM0YfhfIGhC18U= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= diff --git a/types/config.go b/types/config.go index 6213b8e..95ec8d3 100644 --- a/types/config.go +++ b/types/config.go @@ -31,6 +31,9 @@ type AgentTypeConfig struct { Iterations int `yaml:"iterations"` MaxAttempts int `yaml:"max_attempts"` MaxRetries int `yaml:"max_retries"` + // Metadata overlays the global Config.Metadata for this agent type + // (per-key: agent keys win, global-only keys are inherited). + Metadata map[string]string `yaml:"metadata,omitempty"` } // Skill is a named, on-demand instruction set. Its Description is listed in the @@ -65,11 +68,17 @@ type HookConfig struct { // Config holds configuration for creating a new session type Config struct { - Model string `yaml:"model"` - APIKey string `yaml:"api_key"` - BaseURL string `yaml:"base_url"` - LogLevel string `yaml:"log_level"` - Prompt string `yaml:"prompt"` + Model string `yaml:"model"` + APIKey string `yaml:"api_key"` + BaseURL string `yaml:"base_url"` + LogLevel string `yaml:"log_level"` + Prompt string `yaml:"prompt"` + // Metadata is a per-request metadata object attached verbatim to every + // chat-completion request (the OpenAI "metadata" field). Backends such as + // LocalAI use it for per-request flags, e.g. {"enable_thinking": "false"} + // to disable reasoning. Applied to the main session and inherited by + // sub-agents (see AgentTypeConfig.Metadata for per-agent overrides). + Metadata map[string]string `yaml:"metadata,omitempty"` MCPServers map[string]MCPServer `yaml:"mcp_servers"` AgentOptions AgentOptions `yaml:"agent_options"` Agents []AgentTypeConfig `yaml:"agents"`