Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ it reaches 1.0.

## [Unreleased]

### Added

- `lore inscribe --body` (and the `body` field on `lore_inscribe` over MCP)
stores full verbatim source material alongside the summary. `summary`
stays the search key; `lore study` returns the `body` for the deep read.
Bodies survive `archive`/`restore` round-trips.

### Fixed

- Reject invalid `lore catalog --kind` overrides before importing entries.
Expand Down
1 change: 1 addition & 0 deletions docs/generated/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ Store knowledge that transcends the current task — patterns, decisions, resear

| flag | type | default | description |
| --- | --- | --- | --- |
| `--body` | string | `—` | full verbatim source material (e.g. --body "$(cat notes.md)"); summary stays the search key |
| `--informs` | stringArray | `[]` | source entry id (LORE-N, ENTRY-N, or bare N) that informs this entry — repeatable, creates provenance edges |
| `--json` | bool | `false` | emit structured JSON result instead of formatted text |
| `--kind`, `-k` | string | `—` | entry kind (required): idea\|research\|decision\|observation\|principle |
Expand Down
4 changes: 4 additions & 0 deletions docs/generated/mcp.md
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,10 @@ _no arguments_
{
"additionalProperties": false,
"properties": {
"body": {
"description": "full verbatim source material (investigation notes, original doc, transcript). summary is the search key; body is what lore_study returns for the deep read",
"type": "string"
},
"informs": {
"description": "source entry IDs (LORE-N, ENTRY-N, or bare N) that inform this entry — creates informs provenance edges at write-time",
"items": {
Expand Down
10 changes: 10 additions & 0 deletions internal/cli/lore_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,16 @@ func renderStudy(w io.Writer, r *lore.StudyResult, cfg *config.Config) {
fmt.Fprintln(w, " "+strings.Repeat("-", 40))
fmt.Fprintf(w, " %s\n\n", r.Entry.Summary)

// BODY is the full verbatim source material. Printed only when present
// so pre-009 entries (and body-less inscribes) keep the lean layout.
// Not indented per-line: bodies are often multi-line code/notes where
// reflowing would corrupt the content.
if r.Entry.Body != "" {
fmt.Fprintln(w, " BODY")
fmt.Fprintln(w, " "+strings.Repeat("-", 40))
fmt.Fprintf(w, "%s\n\n", r.Entry.Body)
}

if len(r.Linked) > 0 {
fmt.Fprintln(w, " LINKED ENTRIES")
fmt.Fprintln(w, " "+strings.Repeat("-", 40))
Expand Down
6 changes: 3 additions & 3 deletions internal/lore/appraise.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ func bumpAccessCounters(ctx context.Context, db *sql.DB, now time.Time, results
// once so every query that returns *Entry uses the same column order
// (which scanEntry relies on).
const entryColumns = `
e.id, e.project_id, e.topic, e.kind, e.title, e.summary,
e.id, e.project_id, e.topic, e.kind, e.title, e.summary, e.body,
COALESCE(e.tags,''), COALESCE(e.file_path,''),
COALESCE(e.source,''), e.status,
e.valid_days, e.needs_review, COALESCE(e.prompted_by,''),
Expand All @@ -464,7 +464,7 @@ func scanEntry(row interface {
var lastAccessed sql.NullString

if err := row.Scan(
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary,
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body,
&tagsStr, &filePath, &source, &e.Status,
&validDays, &needsReviewInt, &promptedBy,
&createdAt, &updatedAt, &e.AccessCount, &lastAccessed,
Expand Down Expand Up @@ -502,7 +502,7 @@ func scanEntryWithBM25(row interface {
var lastAccessed sql.NullString

if err := row.Scan(
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary,
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body,
&tagsStr, &filePath, &source, &e.Status,
&validDays, &needsReviewInt, &promptedBy,
&createdAt, &updatedAt, &e.AccessCount, &lastAccessed,
Expand Down
5 changes: 3 additions & 2 deletions internal/lore/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type snapshotLoreEntry struct {
Kind string `json:"kind"`
Title string `json:"title"`
Summary string `json:"summary"`
Body string `json:"body,omitempty"`
Tags string `json:"tags,omitempty"`
FilePath string `json:"file_path,omitempty"`
Source string `json:"source,omitempty"`
Expand Down Expand Up @@ -103,7 +104,7 @@ func Archive(ctx context.Context, db *sql.DB, projectID, snapshotPath string) er

// Fetch all entries for this project.
entryRows, err := db.QueryContext(ctx,
`SELECT id, project_id, topic, kind, title, summary,
`SELECT id, project_id, topic, kind, title, summary, body,
COALESCE(tags,''), COALESCE(file_path,''), COALESCE(source,''),
status, valid_days, needs_review,
COALESCE(prompted_by,''), created_at, updated_at, access_count
Expand All @@ -122,7 +123,7 @@ func Archive(ctx context.Context, db *sql.DB, projectID, snapshotPath string) er
var e snapshotLoreEntry
var validDays sql.NullInt64
if err := entryRows.Scan(
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary,
&e.ID, &e.ProjectID, &e.Topic, &e.Kind, &e.Title, &e.Summary, &e.Body,
&e.Tags, &e.FilePath, &e.Source,
&e.Status, &validDays, &e.NeedsReview,
&e.PromptedBy, &e.CreatedAt, &e.UpdatedAt, &e.AccessCount,
Expand Down
7 changes: 5 additions & 2 deletions internal/lore/inscribe.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type InscribeParams struct {
Kind Kind // one of KindIdea..KindPrinciple; required
Title string // required
Summary string // required
Body string // optional full verbatim source material; stored as-is, returned by lore study
Topic string // required
Tags []string // optional semantic tags
Informs []int64 // optional source entry IDs — creates informs edges after insert
Expand Down Expand Up @@ -201,14 +202,15 @@ func Inscribe(ctx context.Context, db *sql.DB, p *InscribeParams) (*InscribeResu
// Prepared INSERT — fixed SQL, bound values.
res, err := db.ExecContext(ctx,
`INSERT INTO entries
(project_id, topic, kind, title, summary, tags, file_path, source,
(project_id, topic, kind, title, summary, body, tags, file_path, source,
status, valid_days, needs_review, prompted_by, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
p.ProjectID,
p.Topic,
string(p.Kind),
p.Title,
p.Summary,
p.Body, // NOT NULL DEFAULT '' — store as-is; "" is the no-body sentinel
nullIfEmpty(tags),
nullIfEmpty(p.FilePath),
nullIfEmpty(p.Source),
Expand Down Expand Up @@ -242,6 +244,7 @@ func Inscribe(ctx context.Context, db *sql.DB, p *InscribeParams) (*InscribeResu
Kind: p.Kind,
Title: p.Title,
Summary: p.Summary,
Body: p.Body,
Tags: append([]string(nil), p.Tags...),
FilePath: p.FilePath,
Source: p.Source,
Expand Down
3 changes: 3 additions & 0 deletions internal/lore/inscribe_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type InscribeInput struct {
Title string `json:"title" jsonschema:"short distinctive title (search-friendly)"`
Kind string `json:"kind" jsonschema:"one of: idea|research|decision|observation|principle"`
Summary string `json:"summary" jsonschema:"2-3 sentence distillation"`
Body string `json:"body,omitempty" jsonschema:"full verbatim source material (investigation notes, original doc, transcript). summary is the search key; body is what lore_study returns for the deep read"`
Topic string `json:"topic" jsonschema:"topic slug (e.g. 'auth', 'caching')"`
Tags []string `json:"tags,omitempty" jsonschema:"semantic tags"`
Informs []string `json:"informs,omitempty" jsonschema:"source entry IDs (LORE-N, ENTRY-N, or bare N) that inform this entry — creates informs provenance edges at write-time"`
Expand All @@ -38,6 +39,7 @@ var InscribeCommand = &command.Command[InscribeInput, InscribeCmdOutput]{
{Name: "title", Kind: command.ArgPositional, Type: command.ArgString, Required: true, Variadic: true, Help: "short distinctive title"},
{Name: "kind", Short: "k", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "entry kind (required): idea|research|decision|observation|principle"},
{Name: "summary", Short: "s", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "2-3 sentence summary (required)"},
{Name: "body", Kind: command.ArgFlag, Type: command.ArgString, Help: "full verbatim source material (e.g. --body \"$(cat notes.md)\"); summary stays the search key"},
{Name: "topic", Short: "t", Kind: command.ArgFlag, Type: command.ArgString, Required: true, Help: "topic slug (required)"},
{Name: "tags", Kind: command.ArgFlag, Type: command.ArgStringSlice, Repeatable: true, Help: "semantic tag (repeatable)"},
{Name: "informs", Kind: command.ArgFlag, Type: command.ArgStringSlice, Repeatable: true, Help: "source entry id (LORE-N, ENTRY-N, or bare N) that informs this entry — repeatable, creates provenance edges"},
Expand Down Expand Up @@ -73,6 +75,7 @@ var InscribeCommand = &command.Command[InscribeInput, InscribeCmdOutput]{
Kind: Kind(in.Kind),
Title: in.Title,
Summary: in.Summary,
Body: in.Body,
Topic: in.Topic,
Tags: in.Tags,
Informs: informs,
Expand Down
69 changes: 69 additions & 0 deletions internal/lore/inscribe_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,75 @@ func TestInscribe_Informs_SelfLink(t *testing.T) {
}
}

// TestInscribe_Body_PersistsAndStudyReturnsIt verifies the verbatim body
// survives the write and comes back intact on the deep read — the whole
// point of the field. summary is the search snippet; body is the payload.
func TestInscribe_Body_PersistsAndStudyReturnsIt(t *testing.T) {
ctx := context.Background()
db := openTestDB(t, "bodyproj")

const fullBody = "# Investigation\n\nTokens expire at exactly 3600s. The refresh\nendpoint returns a new pair; the old refresh token is revoked on use.\nSee the trace in run-4417 for the 401 → 200 sequence."

res, err := Inscribe(ctx, db, &InscribeParams{
ProjectID: "bodyproj",
Kind: KindObservation,
Title: "auth token refresh lifetime",
Summary: "tokens expire at 1h; refresh rotates the pair.",
Body: fullBody,
Topic: "auth",
Now: time.Date(2026, 4, 16, 12, 0, 0, 0, time.UTC),
})
if err != nil {
t.Fatalf("inscribe: %v", err)
}
if res.Entry.Body != fullBody {
t.Errorf("returned Entry.Body = %q, want %q", res.Entry.Body, fullBody)
}

// The read surface (study) must return the full body verbatim.
study, err := Study(ctx, db, res.Entry.ID)
if err != nil {
t.Fatalf("study: %v", err)
}
if study.Entry.Body != fullBody {
t.Errorf("studied Entry.Body = %q, want %q", study.Entry.Body, fullBody)
}
// summary is independent of body — it stays the short distillation.
if study.Entry.Summary == study.Entry.Body {
t.Error("summary and body should be distinct; body must not overwrite summary")
}
}

// TestInscribe_Body_DefaultsEmpty verifies an inscribe with no body leaves
// the column at the empty-string default (pre-009 / no-body behavior) and
// reads back as "" rather than erroring on the NOT NULL column.
func TestInscribe_Body_DefaultsEmpty(t *testing.T) {
ctx := context.Background()
db := openTestDB(t, "bodyproj")

res, err := Inscribe(ctx, db, &InscribeParams{
ProjectID: "bodyproj",
Kind: KindResearch,
Title: "entry with no body supplied",
Summary: "a summary long enough to pass validation.",
Topic: "test",
Now: time.Date(2026, 4, 16, 12, 0, 0, 0, time.UTC),
})
if err != nil {
t.Fatalf("inscribe: %v", err)
}
if res.Entry.Body != "" {
t.Errorf("Entry.Body = %q, want empty string", res.Entry.Body)
}
study, err := Study(ctx, db, res.Entry.ID)
if err != nil {
t.Fatalf("study: %v", err)
}
if study.Entry.Body != "" {
t.Errorf("studied Entry.Body = %q, want empty string", study.Entry.Body)
}
}

// --- tiny utility: contains without importing strings from test helpers ---
func contains(s, sub string) bool {
for i := 0; i+len(sub) <= len(s); i++ {
Expand Down
5 changes: 3 additions & 2 deletions internal/lore/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,15 @@ func restoreV1(ctx context.Context, db *sql.DB, projectID string, data []byte) (

res, err := db.ExecContext(ctx,
`INSERT INTO entries
(project_id, topic, kind, title, summary, tags, file_path, source,
(project_id, topic, kind, title, summary, body, tags, file_path, source,
status, valid_days, needs_review, prompted_by, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
projectID,
e.Topic,
e.Kind,
e.Title,
e.Summary,
e.Body, // NOT NULL DEFAULT '' — empty when restoring a pre-009 snapshot
nullIfEmpty(tags),
nullIfEmpty(filePath),
nullIfEmpty(source),
Expand Down
42 changes: 42 additions & 0 deletions internal/lore/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,3 +317,45 @@ func TestArchiveRestoreRoundTrip(t *testing.T) {
t.Errorf("restored title = %q; want 'Go error handling'", title)
}
}

// TestArchiveRestoreRoundTrip_PreservesBody guards the backup path: the
// verbatim body must survive archive → restore. Without body in the
// snapshot projection or the restore INSERT, a round-trip would silently
// drop the full source material and leave only the summary.
func TestArchiveRestoreRoundTrip_PreservesBody(t *testing.T) {
ctx := context.Background()
db := openTestDB(t, "rtbody-src", "rtbody-dst")

const fullBody = "line one of the verbatim body\nline two with detail\nline three"

res, err := Inscribe(ctx, db, &InscribeParams{
ProjectID: "rtbody-src",
Kind: KindObservation,
Title: "entry carrying a verbatim body",
Summary: "short distillation for the search snippet.",
Body: fullBody,
Topic: "test",
})
if err != nil {
t.Fatalf("inscribe src entry: %v", err)
}
_ = res

snapshotPath := filepath.Join(t.TempDir(), "snapshot.json")
if err := Archive(ctx, db, "rtbody-src", snapshotPath); err != nil {
t.Fatalf("Archive: %v", err)
}
if _, err := Restore(ctx, db, "rtbody-dst", snapshotPath); err != nil {
t.Fatalf("Restore: %v", err)
}

var body string
if err := db.QueryRowContext(ctx,
`SELECT body FROM entries WHERE project_id = 'rtbody-dst'`,
).Scan(&body); err != nil {
t.Fatalf("read restored body: %v", err)
}
if body != fullBody {
t.Errorf("restored body = %q; want %q", body, fullBody)
}
}
3 changes: 3 additions & 0 deletions internal/lore/study_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ func formatStudyResult(s lineSink, o StudyCmdOutput) string {
fmt.Fprintf(&b, " title: %s\n", e.Title)
fmt.Fprintf(&b, " topic: %s\n", e.Topic)
fmt.Fprintf(&b, " summary: %s\n", e.Summary)
if e.Body != "" {
fmt.Fprintf(&b, " body: %s\n", e.Body)
}
if len(e.Tags) > 0 {
fmt.Fprintf(&b, " tags: %s\n", strings.Join(e.Tags, ","))
}
Expand Down
1 change: 1 addition & 0 deletions internal/lore/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type Entry struct {
Kind Kind
Title string
Summary string
Body string // full verbatim source material; "" when none (the inscribe-time default)
Tags []string // parsed from comma-separated DB column; empty slice if NULL
FilePath string // "" if NULL
Source string // "" if NULL
Expand Down
27 changes: 27 additions & 0 deletions internal/storage/migrations/009_lore_body.up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-- 009_lore_body.up.sql
--
-- Verbatim body storage for lore entries.
--
-- Before this migration `entries` carried a `summary` only — the 2-3
-- sentence distillation written at inscribe time. Full source material
-- (an investigation log, the original doc, a pasted transcript) had
-- nowhere to live except `file_path`, a pointer that breaks the moment
-- the file moves or the agent runs somewhere without that filesystem.
--
-- `body` is the inline payload: the full content an agent retrieves with
-- `lore study` after `summary` matched the search. summary stays the
-- search key; body is what you read once you've found the entry.
--
-- NOT indexed into entries_fts. Adding body to the FTS index changes BM25
-- ranking over the whole corpus (longer bodies shift term frequencies),
-- which this project gates on retrieval-quality evals — see migration 004's
-- Recall@5 measurement. That belongs in its own eval-backed change, not
-- bundled with the storage primitive. This migration only adds the column.
--
-- NOT NULL DEFAULT '' so the ALTER backfills every existing row with the
-- empty string — pre-009 entries simply have no body, and the read path
-- treats "" as "no body" (printed only when non-empty).
--
-- Idempotency: schema_migrations prevents re-execution.

ALTER TABLE entries ADD COLUMN body TEXT NOT NULL DEFAULT '';