feat: builtin media tools — read_image / transcribe_audio / read_video

chat/mediatools.go

@@ -0,0 +1,100 @@
+package chat
+
+import (
+	"path/filepath"
+
+	"github.com/mudler/cogito"
+)
+
+// resolveWorkspacePath roots relative paths at workingDir (absolute paths are
+// used as-is, and an empty workingDir leaves the path unchanged). It does not
+// confine paths to the workspace — absolute paths and ".." traversal are not
+// restricted, matching how the host file tools resolve paths.
+func resolveWorkspacePath(workingDir, p string) string {
+	if workingDir == "" || filepath.IsAbs(p) {
+		return p
+	}
+	return filepath.Join(workingDir, p)
+}
+
+// ---- read_image ----
+type readImageArgs struct {
+	Path     string `json:"path" jsonschema:"path to the image file (relative to the workspace or absolute)"`
+	Question string `json:"question,omitempty" jsonschema:"optional specific question about the image; omit for a general description"`
+}
+type readImageTool struct {
+	describe func(path, question string) (string, error)
+}
+
+func (t *readImageTool) Run(args map[string]any) (string, any, error) {
+	path, _ := args["path"].(string)
+	if path == "" {
+		return "read_image error: 'path' is required", nil, nil
+	}
+	question, _ := args["question"].(string)
+	out, err := t.describe(path, question)
+	if err != nil {
+		return "read_image failed: " + err.Error(), nil, nil
+	}
+	return out, nil, nil
+}
+
+func readImageToolDefinition(describe func(path, question string) (string, error)) cogito.ToolDefinitionInterface {
+	return cogito.NewToolDefinition[map[string]any](&readImageTool{describe: describe}, readImageArgs{},
+		"read_image",
+		"Read an image file from the workspace and return a text description of it. Provide `question` to ask something specific about the image; omit it for a general description. Returns model-generated text, not the raw image.")
+}
+
+// ---- transcribe_audio ----
+type transcribeAudioArgs struct {
+	Path string `json:"path" jsonschema:"path to the audio file to transcribe (relative to the workspace or absolute)"`
+}
+type transcribeAudioTool struct {
+	transcribe func(path string) (string, error)
+}
+
+func (t *transcribeAudioTool) Run(args map[string]any) (string, any, error) {
+	path, _ := args["path"].(string)
+	if path == "" {
+		return "transcribe_audio error: 'path' is required", nil, nil
+	}
+	out, err := t.transcribe(path)
+	if err != nil {
+		return "transcribe_audio failed: " + err.Error(), nil, nil
+	}
+	return out, nil, nil
+}
+
+func transcribeAudioToolDefinition(transcribe func(path string) (string, error)) cogito.ToolDefinitionInterface {
+	return cogito.NewToolDefinition[map[string]any](&transcribeAudioTool{transcribe: transcribe}, transcribeAudioArgs{},
+		"transcribe_audio",
+		"Transcribe an audio file from the workspace to text. Returns the transcript.")
+}
+
+// ---- read_video ----
+type readVideoArgs struct {
+	Path     string `json:"path" jsonschema:"path to the video file (relative to the workspace or absolute)"`
+	Question string `json:"question,omitempty" jsonschema:"optional specific question about the video; omit for a general description"`
+}
+type readVideoTool struct {
+	describe func(path, question string) (string, error)
+}
+
+func (t *readVideoTool) Run(args map[string]any) (string, any, error) {
+	path, _ := args["path"].(string)
+	if path == "" {
+		return "read_video error: 'path' is required", nil, nil
+	}
+	question, _ := args["question"].(string)
+	out, err := t.describe(path, question)
+	if err != nil {
+		return "read_video failed: " + err.Error(), nil, nil
+	}
+	return out, nil, nil
+}
+
+func readVideoToolDefinition(describe func(path, question string) (string, error)) cogito.ToolDefinitionInterface {
+	return cogito.NewToolDefinition[map[string]any](&readVideoTool{describe: describe}, readVideoArgs{},
+		"read_video",
+		"Read a video file from the workspace and return a text description of it. Provide `question` to ask something specific; omit for a general description. Returns model-generated text.")
+}

chat/mediatools_test.go

@@ -0,0 +1,116 @@
+package chat
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestResolveWorkspacePath(t *testing.T) {
+	if got := resolveWorkspacePath("/work", "a.png"); got != filepath.Join("/work", "a.png") {
+		t.Fatalf("relative should join workingDir, got %q", got)
+	}
+	if got := resolveWorkspacePath("/work", "/abs/a.png"); got != "/abs/a.png" {
+		t.Fatalf("absolute should be unchanged, got %q", got)
+	}
+	if got := resolveWorkspacePath("", "a.png"); got != "a.png" {
+		t.Fatalf("empty workingDir should leave path, got %q", got)
+	}
+}
+
+func TestReadImageToolRun(t *testing.T) {
+	var gotPath, gotQ string
+	tool := &readImageTool{describe: func(path, question string) (string, error) {
+		gotPath, gotQ = path, question
+		return "a red square", nil
+	}}
+	res, _, err := tool.Run(map[string]any{"path": "x.png", "question": "what color?"})
+	if err != nil || res != "a red square" {
+		t.Fatalf("run: res=%q err=%v", res, err)
+	}
+	if gotPath != "x.png" || gotQ != "what color?" {
+		t.Fatalf("args not threaded: %q %q", gotPath, gotQ)
+	}
+	// empty path → error result, delegate not called
+	res, _, _ = tool.Run(map[string]any{})
+	if res == "a red square" || res == "" {
+		t.Fatalf("empty path should return an error result, got %q", res)
+	}
+	// gotPath must still hold the prior value: the empty-path guard skips the
+	// delegate rather than invoking it with an empty path.
+	if gotPath != "x.png" {
+		t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath)
+	}
+}
+
+func TestTranscribeAudioToolRun(t *testing.T) {
+	var gotPath string
+	tool := &transcribeAudioTool{transcribe: func(path string) (string, error) {
+		gotPath = path
+		return "hello world", nil
+	}}
+	res, _, err := tool.Run(map[string]any{"path": "a.wav"})
+	if err != nil || res != "hello world" {
+		t.Fatalf("run: res=%q err=%v", res, err)
+	}
+	if gotPath != "a.wav" {
+		t.Fatalf("path not threaded: %q", gotPath)
+	}
+	// empty path → error result, delegate not called
+	res, _, _ = tool.Run(map[string]any{})
+	if !strings.Contains(res, "required") {
+		t.Fatalf("empty path should return a 'required' error result, got %q", res)
+	}
+	if gotPath != "a.wav" {
+		t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath)
+	}
+	// delegate error → failed result
+	errTool := &transcribeAudioTool{transcribe: func(path string) (string, error) {
+		return "", errors.New("boom")
+	}}
+	res, _, _ = errTool.Run(map[string]any{"path": "a.wav"})
+	if !strings.Contains(res, "failed") {
+		t.Fatalf("delegate error should return a 'failed' result, got %q", res)
+	}
+}
+
+func TestReadVideoToolRun(t *testing.T) {
+	var gotPath, gotQ string
+	tool := &readVideoTool{describe: func(path, question string) (string, error) {
+		gotPath, gotQ = path, question
+		return "a person waving", nil
+	}}
+	res, _, _ := tool.Run(map[string]any{"path": "c.mp4", "question": "what happens?"})
+	if res != "a person waving" {
+		t.Fatalf("run: %q", res)
+	}
+	if gotPath != "c.mp4" || gotQ != "what happens?" {
+		t.Fatalf("args not threaded: %q %q", gotPath, gotQ)
+	}
+	// empty path → error result, delegate not called
+	res, _, _ = tool.Run(map[string]any{})
+	if !strings.Contains(res, "required") {
+		t.Fatalf("empty path should return a 'required' error result, got %q", res)
+	}
+	if gotPath != "c.mp4" {
+		t.Fatalf("empty-path guard should skip the delegate, but gotPath=%q", gotPath)
+	}
+	// delegate error → failed result
+	errTool := &readVideoTool{describe: func(path, question string) (string, error) {
+		return "", errors.New("boom")
+	}}
+	res, _, _ = errTool.Run(map[string]any{"path": "c.mp4"})
+	if !strings.Contains(res, "failed") {
+		t.Fatalf("delegate error should return a 'failed' result, got %q", res)
+	}
+}
+
+func TestMediaToolsAreReadOnly(t *testing.T) {
+	var noCmds readOnlyCommands
+	for _, n := range []string{"read_image", "transcribe_audio", "read_video"} {
+		if !IsReadOnly(n, "{}", noCmds) {
+			t.Fatalf("%s should be read-only (auto-approve)", n)
+		}
+	}
+}

chat/readonly.go

@@ -15,6 +15,9 @@ var readOnlyTools = map[string]bool{
 	"check_agent":      true,
 	"get_agent_result": true,
 	"cron_list":        true,
+	"read_image":       true,
+	"transcribe_audio": true,
+	"read_video":       true,
 }
 
 // IsReadOnly reports whether a tool call only observes state and is therefore

chat/session.go

@@ -16,6 +16,7 @@ import (
 	"github.com/mudler/nib/manage"
 	wizmcp "github.com/mudler/nib/mcp"
 	"github.com/mudler/nib/plugin"
+	"github.com/mudler/nib/specialist"
 	"github.com/mudler/nib/trace"
 	"github.com/mudler/nib/types"
 
@@ -101,6 +102,8 @@ type Session struct {
 	baseURL         string
 	transcribeModel string
 	visionModel     string
+	videoModel      string
+	workingDir      string
 	metadata        map[string]string // global per-request metadata; merged with per-agent overrides
 	reasoningEffort string            // OpenAI reasoning_effort sent on every request (e.g. "none")
 
@@ -242,6 +245,8 @@ func NewSession(ctx context.Context, cfg types.Config, callbacks Callbacks, tran
 		baseURL:             cfg.BaseURL,
 		transcribeModel:     cfg.TranscribeModel,
 		visionModel:         cfg.VisionModel,
+		videoModel:          cfg.VideoModel,
+		workingDir:          cfg.WorkingDir,
 		metadata:            cfg.Metadata,
 		reasoningEffort:     cfg.ReasoningEffort,
 		mcpClient:           client,
@@ -924,6 +929,31 @@ func (s *Session) SendMessage(text string, parts ...ContentPart) (string, error)
 		})))
 	}
 
+	// Media understanding tools, gated by the allowlist. Each delegates to a
+	// specialist client with the tool's dedicated model and scopes the path to
+	// the session working dir, mirroring how host file tools resolve paths.
+	if s.toolEnabled("read_image") {
+		cogitoOpts = append(cogitoOpts, cogito.WithTools(readImageToolDefinition(
+			func(path, question string) (string, error) {
+				return specialist.New(s.baseURL, s.apiKey).Describe(
+					turnCtx, resolveWorkspacePath(s.workingDir, path), s.visionModel, question)
+			})))
+	}
+	if s.toolEnabled("transcribe_audio") {
+		cogitoOpts = append(cogitoOpts, cogito.WithTools(transcribeAudioToolDefinition(
+			func(path string) (string, error) {
+				return specialist.New(s.baseURL, s.apiKey).Transcribe(
+					turnCtx, resolveWorkspacePath(s.workingDir, path), s.transcribeModel)
+			})))
+	}
+	if s.toolEnabled("read_video") {
+		cogitoOpts = append(cogitoOpts, cogito.WithTools(readVideoToolDefinition(
+			func(path, question string) (string, error) {
+				return specialist.New(s.baseURL, s.apiKey).DescribeVideo(
+					turnCtx, resolveWorkspacePath(s.workingDir, path), s.videoModel, question)
+			})))
+	}
+
 	// Register the goal_done tool only while a goal is active, so it never
 	// appears as a no-op tool in ordinary turns. The callback records
 	// completion: it sets the per-run flag (read by the stop-gate) and clears

specialist/describe.go

@@ -89,3 +89,59 @@ func (c *Client) Describe(ctx context.Context, path, model, prompt string) (stri
 	}
 	return out.Choices[0].Message.Content, nil
 }
+
+// DescribeVideo turns a video into text by sending it to a video-capable model
+// (via a native video_url content part) and returning the model's textual
+// description. Mirrors Describe; it is the building block for the read_video tool.
+// Empty model ⇒ LocalAI's default chat model (which must support video input).
+func (c *Client) DescribeVideo(ctx context.Context, path, model, prompt string) (string, error) {
+	if prompt == "" {
+		prompt = "Describe this video in detail."
+	}
+	uri, err := DataURI(path)
+	if err != nil {
+		return "", err
+	}
+	payload := map[string]any{
+		"model": model,
+		"messages": []map[string]any{{
+			"role": "user",
+			"content": []map[string]any{
+				{"type": "text", "text": prompt},
+				{"type": "video_url", "video_url": map[string]string{"url": uri}},
+			},
+		}},
+	}
+	buf, _ := json.Marshal(payload)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/chat/completions", bytes.NewReader(buf))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if c.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+c.apiKey)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("describe video failed: %s: %s", resp.Status, b)
+	}
+	var out struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return "", err
+	}
+	if len(out.Choices) == 0 {
+		return "", fmt.Errorf("describe video: empty choices")
+	}
+	return out.Choices[0].Message.Content, nil
+}