diff --git a/chat/mediatools.go b/chat/mediatools.go new file mode 100644 index 0000000..7a4ce37 --- /dev/null +++ b/chat/mediatools.go @@ -0,0 +1,100 @@ +package chat + +import ( + "path/filepath" + + "github.com/mudler/cogito" +) + +// resolveWorkspacePath roots relative paths at workingDir (absolute paths are +// used as-is, and an empty workingDir leaves the path unchanged). It does not +// confine paths to the workspace — absolute paths and ".." traversal are not +// restricted, matching how the host file tools resolve paths. +func resolveWorkspacePath(workingDir, p string) string { + if workingDir == "" || filepath.IsAbs(p) { + return p + } + return filepath.Join(workingDir, p) +} + +// ---- read_image ---- +type readImageArgs struct { + Path string `json:"path" jsonschema:"path to the image file (relative to the workspace or absolute)"` + Question string `json:"question,omitempty" jsonschema:"optional specific question about the image; omit for a general description"` +} +type readImageTool struct { + describe func(path, question string) (string, error) +} + +func (t *readImageTool) Run(args map[string]any) (string, any, error) { + path, _ := args["path"].(string) + if path == "" { + return "read_image error: 'path' is required", nil, nil + } + question, _ := args["question"].(string) + out, err := t.describe(path, question) + if err != nil { + return "read_image failed: " + err.Error(), nil, nil + } + return out, nil, nil +} + +func readImageToolDefinition(describe func(path, question string) (string, error)) cogito.ToolDefinitionInterface { + return cogito.NewToolDefinition[map[string]any](&readImageTool{describe: describe}, readImageArgs{}, + "read_image", + "Read an image file from the workspace and return a text description of it. Provide `question` to ask something specific about the image; omit it for a general description. Returns model-generated text, not the raw image.") +} + +// ---- transcribe_audio ---- +type transcribeAudioArgs struct { + Path string `json:"path" jsonschema:"path to the audio file to transcribe (relative to the workspace or absolute)"` +} +type transcribeAudioTool struct { + transcribe func(path string) (string, error) +} + +func (t *transcribeAudioTool) Run(args map[string]any) (string, any, error) { + path, _ := args["path"].(string) + if path == "" { + return "transcribe_audio error: 'path' is required", nil, nil + } + out, err := t.transcribe(path) + if err != nil { + return "transcribe_audio failed: " + err.Error(), nil, nil + } + return out, nil, nil +} + +func transcribeAudioToolDefinition(transcribe func(path string) (string, error)) cogito.ToolDefinitionInterface { + return cogito.NewToolDefinition[map[string]any](&transcribeAudioTool{transcribe: transcribe}, transcribeAudioArgs{}, + "transcribe_audio", + "Transcribe an audio file from the workspace to text. Returns the transcript.") +} + +// ---- read_video ---- +type readVideoArgs struct { + Path string `json:"path" jsonschema:"path to the video file (relative to the workspace or absolute)"` + Question string `json:"question,omitempty" jsonschema:"optional specific question about the video; omit for a general description"` +} +type readVideoTool struct { + describe func(path, question string) (string, error) +} + +func (t *readVideoTool) Run(args map[string]any) (string, any, error) { + path, _ := args["path"].(string) + if path == "" { + return "read_video error: 'path' is required", nil, nil + } + question, _ := args["question"].(string) + out, err := t.describe(path, question) + if err != nil { + return "read_video failed: " + err.Error(), nil, nil + } + return out, nil, nil +} + +func readVideoToolDefinition(describe func(path, question string) (string, error)) cogito.ToolDefinitionInterface { + return cogito.NewToolDefinition[map[string]any](&readVideoTool{describe: describe}, readVideoArgs{}, + "read_video", + "Read a video file from the workspace and return a text description of it. Provide `question` to ask something specific; omit for a general description. Returns model-generated text.") +} diff --git a/chat/mediatools_test.go b/chat/mediatools_test.go new file mode 100644 index 0000000..c346b5b --- /dev/null +++ b/chat/mediatools_test.go @@ -0,0 +1,116 @@ +package chat + +import ( + "errors" + "path/filepath" + "strings" + "testing" +) + +func TestResolveWorkspacePath(t *testing.T) { + if got := resolveWorkspacePath("/work", "a.png"); got != filepath.Join("/work", "a.png") { + t.Fatalf("relative should join workingDir, got %q", got) + } + if got := resolveWorkspacePath("/work", "/abs/a.png"); got != "/abs/a.png" { + t.Fatalf("absolute should be unchanged, got %q", got) + } + if got := resolveWorkspacePath("", "a.png"); got != "a.png" { + t.Fatalf("empty workingDir should leave path, got %q", got) + } +} + +func TestReadImageToolRun(t *testing.T) { + var gotPath, gotQ string + tool := &readImageTool{describe: func(path, question string) (string, error) { + gotPath, gotQ = path, question + return "a red square", nil + }} + res, _, err := tool.Run(map[string]any{"path": "x.png", "question": "what color?"}) + if err != nil || res != "a red square" { + t.Fatalf("run: res=%q err=%v", res, err) + } + if gotPath != "x.png" || gotQ != "what color?" { + t.Fatalf("args not threaded: %q %q", gotPath, gotQ) + } + // empty path → error result, delegate not called + res, _, _ = tool.Run(map[string]any{}) + if res == "a red square" || res == "" { + t.Fatalf("empty path should return an error result, got %q", res) + } + // gotPath must still hold the prior value: the empty-path guard skips the + // delegate rather than invoking it with an empty path. + if gotPath != "x.png" { + t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath) + } +} + +func TestTranscribeAudioToolRun(t *testing.T) { + var gotPath string + tool := &transcribeAudioTool{transcribe: func(path string) (string, error) { + gotPath = path + return "hello world", nil + }} + res, _, err := tool.Run(map[string]any{"path": "a.wav"}) + if err != nil || res != "hello world" { + t.Fatalf("run: res=%q err=%v", res, err) + } + if gotPath != "a.wav" { + t.Fatalf("path not threaded: %q", gotPath) + } + // empty path → error result, delegate not called + res, _, _ = tool.Run(map[string]any{}) + if !strings.Contains(res, "required") { + t.Fatalf("empty path should return a 'required' error result, got %q", res) + } + if gotPath != "a.wav" { + t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath) + } + // delegate error → failed result + errTool := &transcribeAudioTool{transcribe: func(path string) (string, error) { + return "", errors.New("boom") + }} + res, _, _ = errTool.Run(map[string]any{"path": "a.wav"}) + if !strings.Contains(res, "failed") { + t.Fatalf("delegate error should return a 'failed' result, got %q", res) + } +} + +func TestReadVideoToolRun(t *testing.T) { + var gotPath, gotQ string + tool := &readVideoTool{describe: func(path, question string) (string, error) { + gotPath, gotQ = path, question + return "a person waving", nil + }} + res, _, _ := tool.Run(map[string]any{"path": "c.mp4", "question": "what happens?"}) + if res != "a person waving" { + t.Fatalf("run: %q", res) + } + if gotPath != "c.mp4" || gotQ != "what happens?" { + t.Fatalf("args not threaded: %q %q", gotPath, gotQ) + } + // empty path → error result, delegate not called + res, _, _ = tool.Run(map[string]any{}) + if !strings.Contains(res, "required") { + t.Fatalf("empty path should return a 'required' error result, got %q", res) + } + if gotPath != "c.mp4" { + t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath) + } + // delegate error → failed result + errTool := &readVideoTool{describe: func(path, question string) (string, error) { + return "", errors.New("boom") + }} + res, _, _ = errTool.Run(map[string]any{"path": "c.mp4"}) + if !strings.Contains(res, "failed") { + t.Fatalf("delegate error should return a 'failed' result, got %q", res) + } +} + +func TestMediaToolsAreReadOnly(t *testing.T) { + var noCmds readOnlyCommands + for _, n := range []string{"read_image", "transcribe_audio", "read_video"} { + if !IsReadOnly(n, "{}", noCmds) { + t.Fatalf("%s should be read-only (auto-approve)", n) + } + } +} diff --git a/chat/readonly.go b/chat/readonly.go index 258c924..cdf14f2 100644 --- a/chat/readonly.go +++ b/chat/readonly.go @@ -15,6 +15,9 @@ var readOnlyTools = map[string]bool{ "check_agent": true, "get_agent_result": true, "cron_list": true, + "read_image": true, + "transcribe_audio": true, + "read_video": true, } // IsReadOnly reports whether a tool call only observes state and is therefore diff --git a/chat/session.go b/chat/session.go index bd67cf8..9aa152e 100644 --- a/chat/session.go +++ b/chat/session.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/nib/manage" wizmcp "github.com/mudler/nib/mcp" "github.com/mudler/nib/plugin" + "github.com/mudler/nib/specialist" "github.com/mudler/nib/trace" "github.com/mudler/nib/types" @@ -101,6 +102,8 @@ type Session struct { baseURL string transcribeModel string visionModel string + videoModel string + workingDir string metadata map[string]string // global per-request metadata; merged with per-agent overrides reasoningEffort string // OpenAI reasoning_effort sent on every request (e.g. "none") @@ -242,6 +245,8 @@ func NewSession(ctx context.Context, cfg types.Config, callbacks Callbacks, tran baseURL: cfg.BaseURL, transcribeModel: cfg.TranscribeModel, visionModel: cfg.VisionModel, + videoModel: cfg.VideoModel, + workingDir: cfg.WorkingDir, metadata: cfg.Metadata, reasoningEffort: cfg.ReasoningEffort, mcpClient: client, @@ -924,6 +929,31 @@ func (s *Session) SendMessage(text string, parts ...ContentPart) (string, error) }))) } + // Media understanding tools, gated by the allowlist. Each delegates to a + // specialist client with the tool's dedicated model and scopes the path to + // the session working dir, mirroring how host file tools resolve paths. + if s.toolEnabled("read_image") { + cogitoOpts = append(cogitoOpts, cogito.WithTools(readImageToolDefinition( + func(path, question string) (string, error) { + return specialist.New(s.baseURL, s.apiKey).Describe( + turnCtx, resolveWorkspacePath(s.workingDir, path), s.visionModel, question) + }))) + } + if s.toolEnabled("transcribe_audio") { + cogitoOpts = append(cogitoOpts, cogito.WithTools(transcribeAudioToolDefinition( + func(path string) (string, error) { + return specialist.New(s.baseURL, s.apiKey).Transcribe( + turnCtx, resolveWorkspacePath(s.workingDir, path), s.transcribeModel) + }))) + } + if s.toolEnabled("read_video") { + cogitoOpts = append(cogitoOpts, cogito.WithTools(readVideoToolDefinition( + func(path, question string) (string, error) { + return specialist.New(s.baseURL, s.apiKey).DescribeVideo( + turnCtx, resolveWorkspacePath(s.workingDir, path), s.videoModel, question) + }))) + } + // Register the goal_done tool only while a goal is active, so it never // appears as a no-op tool in ordinary turns. The callback records // completion: it sets the per-run flag (read by the stop-gate) and clears diff --git a/specialist/describe.go b/specialist/describe.go index 406ca46..ef81b8c 100644 --- a/specialist/describe.go +++ b/specialist/describe.go @@ -89,3 +89,59 @@ func (c *Client) Describe(ctx context.Context, path, model, prompt string) (stri } return out.Choices[0].Message.Content, nil } + +// DescribeVideo turns a video into text by sending it to a video-capable model +// (via a native video_url content part) and returning the model's textual +// description. Mirrors Describe; it is the building block for the read_video tool. +// Empty model ⇒ LocalAI's default chat model (which must support video input). +func (c *Client) DescribeVideo(ctx context.Context, path, model, prompt string) (string, error) { + if prompt == "" { + prompt = "Describe this video in detail." + } + uri, err := DataURI(path) + if err != nil { + return "", err + } + payload := map[string]any{ + "model": model, + "messages": []map[string]any{{ + "role": "user", + "content": []map[string]any{ + {"type": "text", "text": prompt}, + {"type": "video_url", "video_url": map[string]string{"url": uri}}, + }, + }}, + } + buf, _ := json.Marshal(payload) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/chat/completions", bytes.NewReader(buf)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + resp, err := c.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("describe video failed: %s: %s", resp.Status, b) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + if len(out.Choices) == 0 { + return "", fmt.Errorf("describe video: empty choices") + } + return out.Choices[0].Message.Content, nil +} diff --git a/specialist/describe_video_test.go b/specialist/describe_video_test.go new file mode 100644 index 0000000..a11be00 --- /dev/null +++ b/specialist/describe_video_test.go @@ -0,0 +1,54 @@ +package specialist + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDescribeVideo(t *testing.T) { + var sawVideoPart bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Messages []struct { + Content []struct { + Type string `json:"type"` + VideoURL *struct { + URL string `json:"url"` + } `json:"video_url"` + } `json:"content"` + } `json:"messages"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + for _, m := range req.Messages { + for _, part := range m.Content { + if part.Type == "video_url" && part.VideoURL != nil && strings.HasPrefix(part.VideoURL.URL, "data:video/") { + sawVideoPart = true + } + } + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"a person waving"}}]}`)) + })) + defer srv.Close() + + dir := t.TempDir() + vid := filepath.Join(dir, "clip.mp4") + _ = os.WriteFile(vid, []byte("\x00\x00\x00\x18ftypmp42fakevideo"), 0o644) + + got, err := New(srv.URL, "").DescribeVideo(context.Background(), vid, "", "") + if err != nil { + t.Fatalf("DescribeVideo: %v", err) + } + if !strings.Contains(got, "waving") { + t.Fatalf("unexpected description %q", got) + } + if !sawVideoPart { + t.Fatal("expected a data:video/ video_url part") + } +} diff --git a/types/config.go b/types/config.go index 5e6968b..8fa55f6 100644 --- a/types/config.go +++ b/types/config.go @@ -92,6 +92,7 @@ type Config struct { // by usecase (FLAG_TRANSCRIPT / FLAG_VISION). TranscribeModel string `yaml:"transcribe_model,omitempty" json:"transcribe_model,omitempty"` VisionModel string `yaml:"vision_model,omitempty" json:"vision_model,omitempty"` + VideoModel string `yaml:"video_model,omitempty" json:"video_model,omitempty"` LogLevel string `yaml:"log_level"` Prompt string `yaml:"prompt"` // Metadata is a per-request metadata object attached verbatim to every