diff --git a/CHANGELOG.md b/CHANGELOG.md index fd69aa4..823c9a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ it reaches 1.0. ## [Unreleased] +### Added + +- `lore inscribe --body` (and the `body` field on `lore_inscribe` over MCP) + stores full verbatim source material alongside the summary. `summary` + stays the search key; `lore study` returns the `body` for the deep read. + Bodies survive `archive`/`restore` round-trips. + ### Fixed - Reject invalid `lore catalog --kind` overrides before importing entries. diff --git a/docs/generated/cli.md b/docs/generated/cli.md index 08721f9..3d67944 100644 --- a/docs/generated/cli.md +++ b/docs/generated/cli.md @@ -420,6 +420,7 @@ Store knowledge that transcends the current task — patterns, decisions, resear | flag | type | default | description | | --- | --- | --- | --- | +| `--body` | string | `—` | full verbatim source material (e.g. --body "$(cat notes.md)"); summary stays the search key | | `--informs` | stringArray | `[]` | source entry id (LORE-N, ENTRY-N, or bare N) that informs this entry — repeatable, creates provenance edges | | `--json` | bool | `false` | emit structured JSON result instead of formatted text | | `--kind`, `-k` | string | `—` | entry kind (required): idea\|research\|decision\|observation\|principle | diff --git a/docs/generated/mcp.md b/docs/generated/mcp.md index 95a3367..6170018 100644 --- a/docs/generated/mcp.md +++ b/docs/generated/mcp.md @@ -391,6 +391,10 @@ _no arguments_ { "additionalProperties": false, "properties": { + "body": { + "description": "full verbatim source material (investigation notes, original doc, transcript). summary is the search key; body is what lore_study returns for the deep read", + "type": "string" + }, "informs": { "description": "source entry IDs (LORE-N, ENTRY-N, or bare N) that inform this entry — creates informs provenance edges at write-time", "items": { diff --git a/internal/cli/lore_read.go b/internal/cli/lore_read.go index 5c9c659..155b6e8 100644 --- a/internal/cli/lore_read.go +++ b/internal/cli/lore_read.go @@ -333,6 +333,16 @@ func renderStudy(w io.Writer, r *lore.StudyResult, cfg *config.Config) { fmt.Fprintln(w, " "+strings.Repeat("-", 40)) fmt.Fprintf(w, " %s\n\n", r.Entry.Summary) + // BODY is the full verbatim source material. Printed only when present + // so pre-009 entries (and body-less inscribes) keep the lean layout. + // Not indented per-line: bodies are often multi-line code/notes where + // reflowing would corrupt the content. + if r.Entry.Body != "" { + fmt.Fprintln(w, " BODY") + fmt.Fprintln(w, " "+strings.Repeat("-", 40)) + fmt.Fprintf(w, "%s\n\n", r.Entry.Body) + } + if len(r.Linked) > 0 { fmt.Fprintln(w, " LINKED ENTRIES") fmt.Fprintln(w, " "+strings.Repeat("-", 40)) diff --git a/internal/lore/appraise.go b/internal/lore/appraise.go index 52f158f..895403a 100644 --- a/internal/lore/appraise.go +++ b/internal/lore/appraise.go @@ -445,7 +445,7 @@ func bumpAccessCounters(ctx context.Context, db *sql.DB, now time.Time, results // once so every query that returns *Entry uses the same column order // (which scanEntry relies on). const entryColumns = ` - e.id, e.project_id, e.topic, e.kind, e.title, e.summary, + e.id, e.project_id, e.topic, e.kind, e.title, e.summary, e.body, COALESCE(e.tags,''), COALESCE(e.file_path,''), COALESCE(e.source,''), e.status, e.valid_days, e.needs_review, COALESCE(e.prompted_by,''), @@ -464,7 +464,7 @@ func scanEntry(row interface { var lastAccessed sql.NullString if err := row.Scan( - &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, + &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body, &tagsStr, &filePath, &source, &e.Status, &validDays, &needsReviewInt, &promptedBy, &createdAt, &updatedAt, &e.AccessCount, &lastAccessed, @@ -502,7 +502,7 @@ func scanEntryWithBM25(row interface { var lastAccessed sql.NullString if err := row.Scan( - &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, + &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body, &tagsStr, &filePath, &source, &e.Status, &validDays, &needsReviewInt, &promptedBy, &createdAt, &updatedAt, &e.AccessCount, &lastAccessed, diff --git a/internal/lore/archive.go b/internal/lore/archive.go index cb4afae..e910e5f 100644 --- a/internal/lore/archive.go +++ b/internal/lore/archive.go @@ -26,6 +26,7 @@ type snapshotLoreEntry struct { Kind string `json:"kind"` Title string `json:"title"` Summary string `json:"summary"` + Body string `json:"body,omitempty"` Tags string `json:"tags,omitempty"` FilePath string `json:"file_path,omitempty"` Source string `json:"source,omitempty"` @@ -103,7 +104,7 @@ func Archive(ctx context.Context, db *sql.DB, projectID, snapshotPath string) er // Fetch all entries for this project. entryRows, err := db.QueryContext(ctx, - `SELECT id, project_id, topic, kind, title, summary, + `SELECT id, project_id, topic, kind, title, summary, body, COALESCE(tags,''), COALESCE(file_path,''), COALESCE(source,''), status, valid_days, needs_review, COALESCE(prompted_by,''), created_at, updated_at, access_count @@ -122,7 +123,7 @@ func Archive(ctx context.Context, db *sql.DB, projectID, snapshotPath string) er var e snapshotLoreEntry var validDays sql.NullInt64 if err := entryRows.Scan( - &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, + &e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body, &e.Tags, &e.FilePath, &e.Source, &e.Status, &validDays, &e.NeedsReview, &e.PromptedBy, &e.CreatedAt, &e.UpdatedAt, &e.AccessCount, diff --git a/internal/lore/inscribe.go b/internal/lore/inscribe.go index bbdb986..dc2bc4e 100644 --- a/internal/lore/inscribe.go +++ b/internal/lore/inscribe.go @@ -19,6 +19,7 @@ type InscribeParams struct { Kind Kind // one of KindIdea..KindPrinciple; required Title string // required Summary string // required + Body string // optional full verbatim source material; stored as-is, returned by lore study Topic string // required Tags []string // optional semantic tags Informs []int64 // optional source entry IDs — creates informs edges after insert @@ -201,14 +202,15 @@ func Inscribe(ctx context.Context, db *sql.DB, p *InscribeParams) (*InscribeResu // Prepared INSERT — fixed SQL, bound values. res, err := db.ExecContext(ctx, `INSERT INTO entries - (project_id, topic, kind, title, summary, tags, file_path, source, + (project_id, topic, kind, title, summary, body, tags, file_path, source, status, valid_days, needs_review, prompted_by, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, p.ProjectID, p.Topic, string(p.Kind), p.Title, p.Summary, + p.Body, // NOT NULL DEFAULT '' — store as-is; "" is the no-body sentinel nullIfEmpty(tags), nullIfEmpty(p.FilePath), nullIfEmpty(p.Source), @@ -242,6 +244,7 @@ func Inscribe(ctx context.Context, db *sql.DB, p *InscribeParams) (*InscribeResu Kind: p.Kind, Title: p.Title, Summary: p.Summary, + Body: p.Body, Tags: append([]string(nil), p.Tags...), FilePath: p.FilePath, Source: p.Source, diff --git a/internal/lore/inscribe_cmd.go b/internal/lore/inscribe_cmd.go index c6970a8..e9dcf3d 100644 --- a/internal/lore/inscribe_cmd.go +++ b/internal/lore/inscribe_cmd.go @@ -13,6 +13,7 @@ type InscribeInput struct { Title string `json:"title" jsonschema:"short distinctive title (search-friendly)"` Kind string `json:"kind" jsonschema:"one of: idea|research|decision|observation|principle"` Summary string `json:"summary" jsonschema:"2-3 sentence distillation"` + Body string `json:"body,omitempty" jsonschema:"full verbatim source material (investigation notes, original doc, transcript). summary is the search key; body is what lore_study returns for the deep read"` Topic string `json:"topic" jsonschema:"topic slug (e.g. 'auth', 'caching')"` Tags []string `json:"tags,omitempty" jsonschema:"semantic tags"` Informs []string `json:"informs,omitempty" jsonschema:"source entry IDs (LORE-N, ENTRY-N, or bare N) that inform this entry — creates informs provenance edges at write-time"` @@ -38,6 +39,7 @@ var InscribeCommand = &command.Command[InscribeInput, InscribeCmdOutput]{ {Name: "title", Kind: command.ArgPositional, Type: command.ArgString, Required: true, Variadic: true, Help: "short distinctive title"}, {Name: "kind", Short: "k", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "entry kind (required): idea|research|decision|observation|principle"}, {Name: "summary", Short: "s", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "2-3 sentence summary (required)"}, + {Name: "body", Kind: command.ArgFlag, Type: command.ArgString, Help: "full verbatim source material (e.g. --body \"$(cat notes.md)\"); summary stays the search key"}, {Name: "topic", Short: "t", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "topic slug (required)"}, {Name: "tags", Kind: command.ArgFlag, Type: command.ArgStringSlice, Repeatable: true, Help: "semantic tag (repeatable)"}, {Name: "informs", Kind: command.ArgFlag, Type: command.ArgStringSlice, Repeatable: true, Help: "source entry id (LORE-N, ENTRY-N, or bare N) that informs this entry — repeatable, creates provenance edges"}, @@ -73,6 +75,7 @@ var InscribeCommand = &command.Command[InscribeInput, InscribeCmdOutput]{ Kind: Kind(in.Kind), Title: in.Title, Summary: in.Summary, + Body: in.Body, Topic: in.Topic, Tags: in.Tags, Informs: informs, diff --git a/internal/lore/inscribe_test.go b/internal/lore/inscribe_test.go index 0fb72e5..48012f3 100644 --- a/internal/lore/inscribe_test.go +++ b/internal/lore/inscribe_test.go @@ -470,6 +470,75 @@ func TestInscribe_Informs_SelfLink(t *testing.T) { } } +// TestInscribe_Body_PersistsAndStudyReturnsIt verifies the verbatim body +// survives the write and comes back intact on the deep read — the whole +// point of the field. summary is the search snippet; body is the payload. +func TestInscribe_Body_PersistsAndStudyReturnsIt(t *testing.T) { + ctx := context.Background() + db := openTestDB(t, "bodyproj") + + const fullBody = "# Investigation\n\nTokens expire at exactly 3600s. The refresh\nendpoint returns a new pair; the old refresh token is revoked on use.\nSee the trace in run-4417 for the 401 → 200 sequence." + + res, err := Inscribe(ctx, db, &InscribeParams{ + ProjectID: "bodyproj", + Kind: KindObservation, + Title: "auth token refresh lifetime", + Summary: "tokens expire at 1h; refresh rotates the pair.", + Body: fullBody, + Topic: "auth", + Now: time.Date(2026, 4, 16, 12, 0, 0, 0, time.UTC), + }) + if err != nil { + t.Fatalf("inscribe: %v", err) + } + if res.Entry.Body != fullBody { + t.Errorf("returned Entry.Body = %q, want %q", res.Entry.Body, fullBody) + } + + // The read surface (study) must return the full body verbatim. + study, err := Study(ctx, db, res.Entry.ID) + if err != nil { + t.Fatalf("study: %v", err) + } + if study.Entry.Body != fullBody { + t.Errorf("studied Entry.Body = %q, want %q", study.Entry.Body, fullBody) + } + // summary is independent of body — it stays the short distillation. + if study.Entry.Summary == study.Entry.Body { + t.Error("summary and body should be distinct; body must not overwrite summary") + } +} + +// TestInscribe_Body_DefaultsEmpty verifies an inscribe with no body leaves +// the column at the empty-string default (pre-009 / no-body behavior) and +// reads back as "" rather than erroring on the NOT NULL column. +func TestInscribe_Body_DefaultsEmpty(t *testing.T) { + ctx := context.Background() + db := openTestDB(t, "bodyproj") + + res, err := Inscribe(ctx, db, &InscribeParams{ + ProjectID: "bodyproj", + Kind: KindResearch, + Title: "entry with no body supplied", + Summary: "a summary long enough to pass validation.", + Topic: "test", + Now: time.Date(2026, 4, 16, 12, 0, 0, 0, time.UTC), + }) + if err != nil { + t.Fatalf("inscribe: %v", err) + } + if res.Entry.Body != "" { + t.Errorf("Entry.Body = %q, want empty string", res.Entry.Body) + } + study, err := Study(ctx, db, res.Entry.ID) + if err != nil { + t.Fatalf("study: %v", err) + } + if study.Entry.Body != "" { + t.Errorf("studied Entry.Body = %q, want empty string", study.Entry.Body) + } +} + // --- tiny utility: contains without importing strings from test helpers --- func contains(s, sub string) bool { for i := 0; i+len(sub) <= len(s); i++ { diff --git a/internal/lore/restore.go b/internal/lore/restore.go index 803a18f..83ed20b 100644 --- a/internal/lore/restore.go +++ b/internal/lore/restore.go @@ -143,14 +143,15 @@ func restoreV1(ctx context.Context, db *sql.DB, projectID string, data []byte) ( res, err := db.ExecContext(ctx, `INSERT INTO entries - (project_id, topic, kind, title, summary, tags, file_path, source, + (project_id, topic, kind, title, summary, body, tags, file_path, source, status, valid_days, needs_review, prompted_by, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, projectID, e.Topic, e.Kind, e.Title, e.Summary, + e.Body, // NOT NULL DEFAULT '' — empty when restoring a pre-009 snapshot nullIfEmpty(tags), nullIfEmpty(filePath), nullIfEmpty(source), diff --git a/internal/lore/restore_test.go b/internal/lore/restore_test.go index b250bf5..0c50e06 100644 --- a/internal/lore/restore_test.go +++ b/internal/lore/restore_test.go @@ -317,3 +317,45 @@ func TestArchiveRestoreRoundTrip(t *testing.T) { t.Errorf("restored title = %q; want 'Go error handling'", title) } } + +// TestArchiveRestoreRoundTrip_PreservesBody guards the backup path: the +// verbatim body must survive archive → restore. Without body in the +// snapshot projection or the restore INSERT, a round-trip would silently +// drop the full source material and leave only the summary. +func TestArchiveRestoreRoundTrip_PreservesBody(t *testing.T) { + ctx := context.Background() + db := openTestDB(t, "rtbody-src", "rtbody-dst") + + const fullBody = "line one of the verbatim body\nline two with detail\nline three" + + res, err := Inscribe(ctx, db, &InscribeParams{ + ProjectID: "rtbody-src", + Kind: KindObservation, + Title: "entry carrying a verbatim body", + Summary: "short distillation for the search snippet.", + Body: fullBody, + Topic: "test", + }) + if err != nil { + t.Fatalf("inscribe src entry: %v", err) + } + _ = res + + snapshotPath := filepath.Join(t.TempDir(), "snapshot.json") + if err := Archive(ctx, db, "rtbody-src", snapshotPath); err != nil { + t.Fatalf("Archive: %v", err) + } + if _, err := Restore(ctx, db, "rtbody-dst", snapshotPath); err != nil { + t.Fatalf("Restore: %v", err) + } + + var body string + if err := db.QueryRowContext(ctx, + `SELECT body FROM entries WHERE project_id = 'rtbody-dst'`, + ).Scan(&body); err != nil { + t.Fatalf("read restored body: %v", err) + } + if body != fullBody { + t.Errorf("restored body = %q; want %q", body, fullBody) + } +} diff --git a/internal/lore/study_cmd.go b/internal/lore/study_cmd.go index 85f7f64..f6683e9 100644 --- a/internal/lore/study_cmd.go +++ b/internal/lore/study_cmd.go @@ -72,6 +72,9 @@ func formatStudyResult(s lineSink, o StudyCmdOutput) string { fmt.Fprintf(&b, " title: %s\n", e.Title) fmt.Fprintf(&b, " topic: %s\n", e.Topic) fmt.Fprintf(&b, " summary: %s\n", e.Summary) + if e.Body != "" { + fmt.Fprintf(&b, " body: %s\n", e.Body) + } if len(e.Tags) > 0 { fmt.Fprintf(&b, " tags: %s\n", strings.Join(e.Tags, ",")) } diff --git a/internal/lore/types.go b/internal/lore/types.go index 1ed38e2..fd37f3c 100644 --- a/internal/lore/types.go +++ b/internal/lore/types.go @@ -62,6 +62,7 @@ type Entry struct { Kind Kind Title string Summary string + Body string // full verbatim source material; "" when none (the inscribe-time default) Tags []string // parsed from comma-separated DB column; empty slice if NULL FilePath string // "" if NULL Source string // "" if NULL diff --git a/internal/storage/migrations/009_lore_body.up.sql b/internal/storage/migrations/009_lore_body.up.sql new file mode 100644 index 0000000..5ac2225 --- /dev/null +++ b/internal/storage/migrations/009_lore_body.up.sql @@ -0,0 +1,27 @@ +-- 009_lore_body.up.sql +-- +-- Verbatim body storage for lore entries. +-- +-- Before this migration `entries` carried a `summary` only — the 2-3 +-- sentence distillation written at inscribe time. Full source material +-- (an investigation log, the original doc, a pasted transcript) had +-- nowhere to live except `file_path`, a pointer that breaks the moment +-- the file moves or the agent runs somewhere without that filesystem. +-- +-- `body` is the inline payload: the full content an agent retrieves with +-- `lore study` after `summary` matched the search. summary stays the +-- search key; body is what you read once you've found the entry. +-- +-- NOT indexed into entries_fts. Adding body to the FTS index changes BM25 +-- ranking over the whole corpus (longer bodies shift term frequencies), +-- which this project gates on retrieval-quality evals — see migration 004's +-- Recall@5 measurement. That belongs in its own eval-backed change, not +-- bundled with the storage primitive. This migration only adds the column. +-- +-- NOT NULL DEFAULT '' so the ALTER backfills every existing row with the +-- empty string — pre-009 entries simply have no body, and the read path +-- treats "" as "no body" (printed only when non-empty). +-- +-- Idempotency: schema_migrations prevents re-execution. + +ALTER TABLE entries ADD COLUMN body TEXT NOT NULL DEFAULT '';