diff --git a/README.md b/README.md index 298a495..b804fb2 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,80 @@ The `store.Store` interface is backend-agnostic. Swap `sqlite.New` for any implementation that satisfies the interface to use a different storage engine without changing callers. +### Path B: document ingestion + +Path B ingests existing Markdown document trees into lore entries. The +ingester is a pure functional transform: it returns entries and the caller +writes them to a Store. + +```go +import ( + "context" + "log" + + "github.com/mathomhaus/lore/pkg/lore/ingest/heuristic" +) + +func main() { + ing := heuristic.NewIngester() + + result, err := ing.Process(context.Background(), "/workspace/docs") + if err != nil { + log.Fatal(err) + } + + for _, fe := range result.Errors { + log.Printf("warn: %v", fe) + } + + for _, e := range result.Entries { + log.Printf("entry kind=%s title=%q source=%s", e.Kind, e.Title, e.Source) + // write e to your store here + } +} +``` + +#### Classification priority + +The heuristic ingester classifies each chunk using this priority order (first +match wins): + +1. YAML front matter with an explicit `kind:` field. +2. Path rules: `docs/adr/*.md` maps to decision+adr; `docs/runbooks/*.md` + maps to procedure+runbook; `CLAUDE.md`/`agents.md`/`skills.md` map to + reference+agent-config; and so on. See `heuristic.DefaultRules()`. +3. Heading keywords: `## What is` maps to explanation; `## Decision` / + `## Context` / `## Consequences` maps to decision; `## Procedure` / + `## Steps` maps to procedure; etc. +4. Fallback: kind=research (catch-all). + +#### Customizing rules + +```go +import "github.com/mathomhaus/lore/pkg/lore/ingest/heuristic" + +rules := heuristic.DefaultRules() +rules = append(rules, heuristic.Rule{ + PathGlob: "docs/specs/*.md", + Kind: lore.KindDecision, + Tags: []string{"spec"}, +}) + +ing := heuristic.NewIngester( + heuristic.WithRules(rules), + heuristic.WithLogger(slog.Default()), +) +``` + +#### Walker behavior (v0.1.1) + +- Only `.md` and `.markdown` files are processed. +- `.git/`, `node_modules/`, `vendor/`, and any hidden directory (name + starting with `.`) are skipped unconditionally. +- Files larger than 10 MB are skipped with a FileError. +- Symlinks are not followed. +- `.gitignore` patterns are not honored (planned for v0.2). + ## Status: pre-v1.0 Lore is pre-v1.0. The exported surface is stable in shape but may change in diff --git a/go.mod b/go.mod index 75c3018..1930913 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 + gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.50.0 ) diff --git a/go.sum b/go.sum index 4e5c03c..a3b088b 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -27,6 +31,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2 h1:jW34+ipjoob+BfuLfALS6FBJh84KJcmzFowmxsImA/o= github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2/go.mod h1:jsOuI9QymOruQ9UdLrCbMPFykhI13d5FcuV60R1wJ6Q= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -48,6 +54,9 @@ golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= diff --git a/pkg/lore/ingest/chunker.go b/pkg/lore/ingest/chunker.go new file mode 100644 index 0000000..ccdd1d3 --- /dev/null +++ b/pkg/lore/ingest/chunker.go @@ -0,0 +1,209 @@ +package ingest + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// Frontmatter holds the fields we parse from a YAML front matter block. +// All fields are optional; unrecognized keys are silently ignored. +type Frontmatter struct { + Kind string `yaml:"kind"` + Tags []string `yaml:"tags"` +} + +// Chunk is an intermediate representation of one heading-bounded section of +// a Markdown file. It is enriched into a lore.Entry by the classifier. +type Chunk struct { + // Title is the text of the heading that opened this chunk, or the file + // stem when the file has no headings. + Title string + + // Body is the raw Markdown for this chunk, including the opening heading + // line. + Body string + + // Source is ":" of the heading. + Source string + + // FM is the parsed front matter from the file. It is identical for + // every chunk derived from the same file; per-chunk overrides are not + // supported in v0.1.1. + FM Frontmatter + + // FilePath is the absolute path of the source file. + FilePath string +} + +// ChunkFile reads the Markdown file at path, extracts any YAML front matter, +// and splits the remainder at H1/H2/H3 boundaries. Each boundary produces one +// Chunk. If the file has no headings, the whole body is returned as a single +// Chunk whose title is the filename stem. +// +// The Source field on each chunk is set to ":" where +// relPath is path relative to root. If root is empty or path is not under +// root, the base name is used instead. +func ChunkFile(path, root string) ([]Chunk, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("ingest: chunker: open %s: %w", path, err) + } + defer func() { _ = f.Close() }() + + relPath := RelativeOrBase(path, root) + + scanner := bufio.NewScanner(f) + + var fm Frontmatter + var bodyLines []string + + lineNum := 0 + inFrontmatter := false + var fmLines []string + fmDone := false + bodyLineOffset := 0 + + for scanner.Scan() { + lineNum++ + line := scanner.Text() + + if lineNum == 1 && line == "---" { + inFrontmatter = true + continue + } + + if inFrontmatter { + if line == "---" || line == "..." { + inFrontmatter = false + fmDone = true + bodyLineOffset = lineNum + continue + } + fmLines = append(fmLines, line) + continue + } + + if !fmDone { + bodyLineOffset = lineNum - 1 + fmDone = true + } + + bodyLines = append(bodyLines, line) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("ingest: chunker: scan %s: %w", path, err) + } + + if len(fmLines) > 0 { + _ = yaml.Unmarshal([]byte(strings.Join(fmLines, "\n")), &fm) + } + + // Absolute line number of the first body line. + firstBodyLine := bodyLineOffset + 1 + + // Find all H1/H2/H3 boundary positions. + type boundary struct { + lineIdx int + title string + } + + var boundaries []boundary + for i, line := range bodyLines { + if title, ok := ExtractHeading(line); ok { + boundaries = append(boundaries, boundary{lineIdx: i, title: title}) + } + } + + fileTitle := stemName(filepath.Base(path)) + + // No headings: whole body is one chunk. + if len(boundaries) == 0 { + body := strings.Join(bodyLines, "\n") + src := fmt.Sprintf("%s:%d", relPath, firstBodyLine) + if strings.TrimSpace(body) == "" { + return nil, nil + } + return []Chunk{{ + Title: fileTitle, + Body: body, + Source: src, + FM: fm, + FilePath: path, + }}, nil + } + + chunks := make([]Chunk, 0, len(boundaries)) + for idx, b := range boundaries { + var endIdx int + if idx+1 < len(boundaries) { + endIdx = boundaries[idx+1].lineIdx + } else { + endIdx = len(bodyLines) + } + + chunkBodyLines := bodyLines[b.lineIdx:endIdx] + body := strings.Join(chunkBodyLines, "\n") + absLineNum := firstBodyLine + b.lineIdx + src := fmt.Sprintf("%s:%d", relPath, absLineNum) + + chunks = append(chunks, Chunk{ + Title: b.title, + Body: body, + Source: src, + FM: fm, + FilePath: path, + }) + } + + return chunks, nil +} + +// ExtractHeading returns the heading text and true if line is an ATX-style +// H1, H2, or H3 Markdown heading (one, two, or three leading # characters +// followed by a space and non-empty text). Returns ("", false) for anything +// else. +func ExtractHeading(line string) (string, bool) { + if len(line) < 3 || line[0] != '#' { + return "", false + } + level := 0 + for level < len(line) && line[level] == '#' { + level++ + } + if level > 3 || level >= len(line) || line[level] != ' ' { + return "", false + } + title := strings.TrimSpace(line[level+1:]) + if title == "" { + return "", false + } + return title, true +} + +// stemName returns the filename without its extension. +func stemName(base string) string { + ext := filepath.Ext(base) + if ext == "" { + return base + } + return base[:len(base)-len(ext)] +} + +// RelativeOrBase returns path relative to root. If the path is not under root +// or Rel returns an error, it falls back to filepath.Base(path). +func RelativeOrBase(path, root string) string { + if root == "" { + return filepath.Base(path) + } + rel, err := filepath.Rel(root, path) + if err != nil || strings.HasPrefix(rel, "..") { + return filepath.Base(path) + } + return rel +} diff --git a/pkg/lore/ingest/heuristic/heuristic.go b/pkg/lore/ingest/heuristic/heuristic.go new file mode 100644 index 0000000..41dfb8c --- /dev/null +++ b/pkg/lore/ingest/heuristic/heuristic.go @@ -0,0 +1,248 @@ +// Package heuristic provides the default Path B heuristic classifier for lore +// ingestion. It implements [ingest.Ingester] using a rule-priority stack: +// +// 1. YAML front matter with explicit kind and tags fields: use as-is. +// 2. Path rules: filepath.Match-style globs against the file path. +// 3. Heading patterns: keywords in H1/H2/H3 heading text. +// 4. Fallback: kind=research (catch-all). +// +// The classifier is configurable via functional options. The zero-value rule +// set (from [DefaultRules]) covers the most common documentation patterns; use +// [WithRules] to replace it entirely. +package heuristic + +import ( + "context" + "log/slog" + "path/filepath" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/mathomhaus/lore/pkg/lore" + "github.com/mathomhaus/lore/pkg/lore/ingest" +) + +const tracerName = "lore/ingest/heuristic" + +// ingester is the concrete implementation of [ingest.Ingester]. +type ingester struct { + rules []Rule + log *slog.Logger + tracer trace.Tracer + walker ingest.WalkerConfig +} + +// Option is a functional option for [NewIngester]. +type Option func(*ingester) + +// WithRules replaces the default classification rule table with the supplied +// slice. Rules are tested in order; the first match wins. +func WithRules(rules []Rule) Option { + return func(i *ingester) { + i.rules = rules + } +} + +// WithLogger sets the structured logger. When nil, logging is discarded. +func WithLogger(l *slog.Logger) Option { + return func(i *ingester) { + if l != nil { + i.log = l + } + } +} + +// WithTracer overrides the OTel tracer used for spans. When nil, the global +// tracer provider is used via [otel.Tracer]. +func WithTracer(t trace.Tracer) Option { + return func(i *ingester) { + if t != nil { + i.tracer = t + } + } +} + +// WithMaxFileSize sets the per-file size limit above which a file is skipped. +// Zero or negative resets to the default (10 MB). +func WithMaxFileSize(n int64) Option { + return func(i *ingester) { + i.walker.MaxFileSize = n + } +} + +// NewIngester returns a heuristic [ingest.Ingester] configured by opts. +// When no [WithRules] option is supplied, [DefaultRules] is used. +func NewIngester(opts ...Option) ingest.Ingester { + ing := &ingester{ + rules: DefaultRules(), + log: slog.Default(), + tracer: otel.Tracer(tracerName), + } + for _, o := range opts { + o(ing) + } + return ing +} + +// Process implements [ingest.Ingester]. It walks root, chunks each Markdown +// file, classifies each chunk, and returns the results. It does not write to +// any store. +func (ing *ingester) Process(ctx context.Context, root string) (ingest.Result, error) { + ctx, span := ing.tracer.Start(ctx, "lore.ingest.process", + trace.WithAttributes(attribute.String("root", root)), + ) + defer span.End() + + wr := ingest.WalkDir(root, ing.walker) + + ing.log.InfoContext(ctx, "ingest: walk complete", + "root", root, + "files", len(wr.Paths), + "walk_errors", len(wr.Errors), + ) + + span.SetAttributes(attribute.Int("path.count", len(wr.Paths))) + + var result ingest.Result + result.Errors = append(result.Errors, wr.Errors...) + + for _, path := range wr.Paths { + entries, ferr := ing.processFile(ctx, path, root) + if ferr != nil { + ing.log.WarnContext(ctx, "ingest: file error", "path", path, "err", ferr) + result.Errors = append(result.Errors, ingest.FileError{Path: path, Err: ferr}) + continue + } + result.Entries = append(result.Entries, entries...) + } + + span.SetAttributes(attribute.Int("entries.count", len(result.Entries))) + ing.log.InfoContext(ctx, "ingest: process complete", + "entries", len(result.Entries), + "file_errors", len(result.Errors), + ) + + return result, nil +} + +// processFile chunks and classifies a single file. +func (ing *ingester) processFile(ctx context.Context, path, root string) ([]lore.Entry, error) { + _, span := ing.tracer.Start(ctx, "lore.ingest.classify", + trace.WithAttributes(attribute.String("file", path)), + ) + defer span.End() + + chunks, err := ingest.ChunkFile(path, root) + if err != nil { + return nil, err + } + + now := time.Now().UTC() + entries := make([]lore.Entry, 0, len(chunks)) + for _, c := range chunks { + kind, tags := ing.classify(c, path, root) + entries = append(entries, lore.Entry{ + Kind: kind, + Title: c.Title, + Body: c.Body, + Source: c.Source, + Tags: tags, + CreatedAt: now, + UpdatedAt: now, + }) + } + + return entries, nil +} + +// classify determines the Kind and Tags for a chunk using the four-level +// priority stack described in the package doc. +func (ing *ingester) classify(c ingest.Chunk, absPath, root string) (lore.Kind, []string) { + // 1. YAML front matter with explicit kind. + if c.FM.Kind != "" { + k := lore.Kind(c.FM.Kind) + if err := k.Validate(); err == nil { + return k, dedupeStrings(c.FM.Tags) + } + // Invalid kind in frontmatter: fall through to path rules. + } + + // 2. Path rules. Test both the repo-relative path and the base name. + relPath := ingest.RelativeOrBase(absPath, root) + base := filepath.Base(absPath) + + for _, rule := range ing.rules { + if matched, _ := filepath.Match(rule.PathGlob, relPath); matched { + return rule.Kind, dedupeStrings(rule.Tags) + } + if matched, _ := filepath.Match(rule.PathGlob, base); matched { + return rule.Kind, dedupeStrings(rule.Tags) + } + } + + // 3. Heading patterns. + if kind, ok := classifyByHeadings(c.Body); ok { + return kind, nil + } + + // 4. Fallback. + return lore.KindResearch, nil +} + +// classifyByHeadings scans the headings in body for classification keywords. +// Returns (kind, true) on the first match. +func classifyByHeadings(body string) (lore.Kind, bool) { + for _, line := range strings.Split(body, "\n") { + title, ok := ingest.ExtractHeading(line) + if !ok { + continue + } + lower := strings.ToLower(title) + + switch { + case containsAny(lower, "procedure", "how to", "how-to", "steps", "runbook", "playbook"): + return lore.KindProcedure, true + case containsAny(lower, "decision", "context", "consequences", "status", "adr"): + return lore.KindDecision, true + case containsAny(lower, "what is", "what are", "overview", "introduction", "concept", "explanation"): + return lore.KindExplanation, true + case containsAny(lower, "principle", "rule", "guideline", "standard"): + return lore.KindPrinciple, true + case containsAny(lower, "observation", "finding", "measurement", "postmortem"): + return lore.KindObservation, true + case containsAny(lower, "research", "spike", "investigation", "exploration"): + return lore.KindResearch, true + } + } + return "", false +} + +// containsAny reports whether s contains any of the supplied substrings. +func containsAny(s string, needles ...string) bool { + for _, n := range needles { + if strings.Contains(s, n) { + return true + } + } + return false +} + +// dedupeStrings returns a deduplicated copy of ss preserving order. +func dedupeStrings(ss []string) []string { + if len(ss) == 0 { + return nil + } + seen := make(map[string]struct{}, len(ss)) + out := make([]string, 0, len(ss)) + for _, s := range ss { + if _, ok := seen[s]; !ok { + seen[s] = struct{}{} + out = append(out, s) + } + } + return out +} diff --git a/pkg/lore/ingest/heuristic/rules.go b/pkg/lore/ingest/heuristic/rules.go new file mode 100644 index 0000000..6d98a73 --- /dev/null +++ b/pkg/lore/ingest/heuristic/rules.go @@ -0,0 +1,78 @@ +package heuristic + +import "github.com/mathomhaus/lore/pkg/lore" + +// Rule is a single path-based classification rule. When a walked file's +// path matches PathGlob (tested against both the repo-relative path and the +// base name), the file is assigned Kind and Tags without further inspection. +// Rules are tested in the order returned by [DefaultRules]; the first match +// wins. +type Rule struct { + // PathGlob is a filepath.Match-style pattern. It is tested against the + // repo-relative path and, as a convenience, also against the file's base + // name alone so patterns like "CLAUDE.md" work regardless of nesting. + PathGlob string + + // Kind is the lore kind assigned to chunks from matching files. + Kind lore.Kind + + // Tags is the list of tags assigned to chunks from matching files. The + // classifier deduplicates the slice before setting it on the entry. + Tags []string +} + +// DefaultRules returns the default rule table. Rules are tested in the order +// returned; the first match wins. Callers may pass a modified copy to +// [WithRules] to extend or replace these defaults. +// +// Default coverage: +// - docs/adr/** ADRs decision + adr +// - docs/runbooks/** Runbooks procedure + runbook +// - docs/playbooks/** Playbooks procedure + playbook +// - docs/concepts/** Concept pages explanation + concept +// - docs/reference/** Reference pages reference +// - agents.md, skills.md, CLAUDE.md reference + agent-config +// - CONTRIBUTING.md procedure + contributing +// - CHANGELOG.md observation + changelog +// - *.md in project root reference (catch-wide-root) +func DefaultRules() []Rule { + return []Rule{ + // ADRs — architectural decision records. + {PathGlob: "docs/adr/*.md", Kind: lore.KindDecision, Tags: []string{"adr"}}, + {PathGlob: "docs/adr/*.markdown", Kind: lore.KindDecision, Tags: []string{"adr"}}, + {PathGlob: "docs/decisions/*.md", Kind: lore.KindDecision, Tags: []string{"adr"}}, + {PathGlob: "docs/decisions/*.markdown", Kind: lore.KindDecision, Tags: []string{"adr"}}, + + // Runbooks. + {PathGlob: "docs/runbooks/*.md", Kind: lore.KindProcedure, Tags: []string{"runbook"}}, + {PathGlob: "docs/runbooks/*.markdown", Kind: lore.KindProcedure, Tags: []string{"runbook"}}, + + // Playbooks. + {PathGlob: "docs/playbooks/*.md", Kind: lore.KindProcedure, Tags: []string{"playbook"}}, + {PathGlob: "docs/playbooks/*.markdown", Kind: lore.KindProcedure, Tags: []string{"playbook"}}, + + // Concept and explanation pages. + {PathGlob: "docs/concepts/*.md", Kind: lore.KindExplanation, Tags: []string{"concept"}}, + {PathGlob: "docs/concepts/*.markdown", Kind: lore.KindExplanation, Tags: []string{"concept"}}, + + // Reference pages. + {PathGlob: "docs/reference/*.md", Kind: lore.KindReference, Tags: []string{"reference"}}, + {PathGlob: "docs/reference/*.markdown", Kind: lore.KindReference, Tags: []string{"reference"}}, + + // Agent configuration files (base-name matching). + {PathGlob: "agents.md", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + {PathGlob: "agents.markdown", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + {PathGlob: "skills.md", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + {PathGlob: "skills.markdown", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + {PathGlob: "CLAUDE.md", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + {PathGlob: "AGENTS.md", Kind: lore.KindReference, Tags: []string{"agent-config"}}, + + // Contributing guide. + {PathGlob: "CONTRIBUTING.md", Kind: lore.KindProcedure, Tags: []string{"contributing"}}, + {PathGlob: "CONTRIBUTING.markdown", Kind: lore.KindProcedure, Tags: []string{"contributing"}}, + + // Changelog — records of what changed. + {PathGlob: "CHANGELOG.md", Kind: lore.KindObservation, Tags: []string{"changelog"}}, + {PathGlob: "CHANGELOG.markdown", Kind: lore.KindObservation, Tags: []string{"changelog"}}, + } +} diff --git a/pkg/lore/ingest/ingest.go b/pkg/lore/ingest/ingest.go new file mode 100644 index 0000000..0bfe21e --- /dev/null +++ b/pkg/lore/ingest/ingest.go @@ -0,0 +1,85 @@ +// Package ingest provides Path B document ingestion for the lore library. +// +// The Ingester interface walks a filesystem directory tree, chunks recognized +// document files (Markdown for v0.1.1), classifies each chunk into a lore +// entry, and returns the entries to the caller. Ingester is a pure functional +// transform: Process returns [Result] and the caller decides how to write the +// entries to a [lore.Store] and [lore.VectorStore]. +// +// # Composing with a store +// +// The caller owns the write side so it can apply its own transactional logic, +// deduplication, or batching: +// +// result, err := ing.Process(ctx, "/workspace/docs") +// if err != nil { +// return err +// } +// for _, e := range result.Entries { +// if _, err := store.Put(ctx, e); err != nil { +// log.Error("put entry", "err", err, "title", e.Title) +// } +// } +// +// # Versioning notes +// +// v0.1.1: Markdown only (.md, .markdown). Filesystem walker only (no git +// index, no incremental). .gitignore patterns NOT honored (v0.2 followup). +// Symlinks are NOT followed. +package ingest + +import ( + "context" + "fmt" + + "github.com/mathomhaus/lore/pkg/lore" +) + +// Ingester walks document trees and produces classified entries. +// Implementations are stateless functional transforms: Process returns the +// entries it computed and the caller is responsible for writing them to a +// Store and VectorStore. +type Ingester interface { + // Process walks root (a filesystem directory), chunks recognized files, + // classifies each chunk, and returns the entries. It does not write to + // any store. Returns a Result with successful entries plus any per-file + // errors that did not abort the whole walk. + // + // A non-nil error from Process signals a fatal failure (e.g. root does + // not exist or is not a directory). Per-file failures that do not abort + // the walk are collected in Result.Errors instead. + Process(ctx context.Context, root string) (Result, error) +} + +// Result carries the output of a single [Ingester.Process] call. +type Result struct { + // Entries are the successfully classified chunks from all walked files. + // Ordered by discovery: file walk order, then chunk order within each file. + Entries []lore.Entry + + // Errors collects per-file failures that did not abort the whole walk. + // A caller that needs strict mode should inspect this slice and decide + // whether to discard Entries or surface the errors. + Errors []FileError +} + +// FileError records a per-file failure during a walk. The walk continues +// after recording the error. +type FileError struct { + // Path is the absolute path of the file that caused the failure. + Path string + + // Err is the underlying error. + Err error +} + +// Error implements the error interface so FileError values can be passed to +// structured loggers that accept error. +func (e FileError) Error() string { + return fmt.Sprintf("ingest: %s: %v", e.Path, e.Err) +} + +// Unwrap returns the underlying error for errors.Is and errors.As chains. +func (e FileError) Unwrap() error { + return e.Err +} diff --git a/pkg/lore/ingest/ingest_test.go b/pkg/lore/ingest/ingest_test.go new file mode 100644 index 0000000..9af1d15 --- /dev/null +++ b/pkg/lore/ingest/ingest_test.go @@ -0,0 +1,366 @@ +package ingest_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/mathomhaus/lore/pkg/lore" + "github.com/mathomhaus/lore/pkg/lore/ingest" + "github.com/mathomhaus/lore/pkg/lore/ingest/heuristic" +) + +// testdataDir returns the absolute path to the testdata directory. +func testdataDir(t *testing.T) string { + t.Helper() + // __file__ is relative to the package dir; testdata is a sibling. + dir, err := filepath.Abs("testdata") + if err != nil { + t.Fatalf("testdata abs path: %v", err) + } + return dir +} + +// newIngester builds a default heuristic ingester for tests. +func newIngester() ingest.Ingester { + return heuristic.NewIngester() +} + +// findEntry returns the first entry in result whose Title contains substr. +func findEntry(entries []lore.Entry, substr string) (lore.Entry, bool) { + for _, e := range entries { + if containsStr(e.Title, substr) { + return e, true + } + } + return lore.Entry{}, false +} + +func containsStr(s, sub string) bool { + return len(s) >= len(sub) && (s == sub || len(sub) == 0 || + func() bool { + for i := 0; i <= len(s)-len(sub); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false + }()) +} + +// hasTag reports whether e has the given tag. +func hasTag(e lore.Entry, tag string) bool { + for _, t := range e.Tags { + if t == tag { + return true + } + } + return false +} + +// TestProcess_Frontmatter verifies that explicit kind/tags in YAML front +// matter take precedence over path rules and heading patterns. +func TestProcess_Frontmatter(t *testing.T) { + root := testdataDir(t) + ing := newIngester() + + result, err := ing.Process(context.Background(), root) + if err != nil { + t.Fatalf("Process: %v", err) + } + + // The ADR file has frontmatter kind=decision, tags=[adr, language]. + e, ok := findEntry(result.Entries, "Use Go for the implementation") + if !ok { + t.Fatalf("expected entry with title 'Use Go for the implementation'; entries: %v", titles(result.Entries)) + } + if e.Kind != lore.KindDecision { + t.Errorf("kind: got %q, want %q", e.Kind, lore.KindDecision) + } + if !hasTag(e, "adr") { + t.Errorf("expected tag 'adr'; tags: %v", e.Tags) + } + if !hasTag(e, "language") { + t.Errorf("expected tag 'language'; tags: %v", e.Tags) + } +} + +// TestProcess_PathRules verifies that path-based rules classify files +// correctly even without explicit front matter. +func TestProcess_PathRules(t *testing.T) { + root := testdataDir(t) + ing := newIngester() + + result, err := ing.Process(context.Background(), root) + if err != nil { + t.Fatalf("Process: %v", err) + } + + // docs/runbooks/deploy.md should be kind=procedure + tag=runbook. + e, ok := findEntry(result.Entries, "Deploy to production") + if !ok { + t.Fatalf("expected entry 'Deploy to production'; entries: %v", titles(result.Entries)) + } + if e.Kind != lore.KindProcedure { + t.Errorf("deploy kind: got %q, want %q", e.Kind, lore.KindProcedure) + } + if !hasTag(e, "runbook") { + t.Errorf("deploy: expected tag 'runbook'; tags: %v", e.Tags) + } + + // CLAUDE.md should be kind=reference + tag=agent-config. + c, ok := findEntry(result.Entries, "lore agent bootstrap") + if !ok { + t.Fatalf("expected entry 'lore agent bootstrap'; entries: %v", titles(result.Entries)) + } + if c.Kind != lore.KindReference { + t.Errorf("CLAUDE.md kind: got %q, want %q", c.Kind, lore.KindReference) + } + if !hasTag(c, "agent-config") { + t.Errorf("CLAUDE.md: expected tag 'agent-config'; tags: %v", c.Tags) + } +} + +// TestProcess_HeadingPatterns verifies that files without explicit front +// matter or matching path rules are classified by heading keywords. +func TestProcess_HeadingPatterns(t *testing.T) { + root := testdataDir(t) + ing := newIngester() + + result, err := ing.Process(context.Background(), root) + if err != nil { + t.Fatalf("Process: %v", err) + } + + // docs/concepts/auth.md has "## What is auth" which signals explanation. + e, ok := findEntry(result.Entries, "What is auth") + if !ok { + t.Fatalf("expected entry 'What is auth'; entries: %v", titles(result.Entries)) + } + if e.Kind != lore.KindExplanation { + t.Errorf("auth concept kind: got %q, want %q", e.Kind, lore.KindExplanation) + } +} + +// TestProcess_Fallback verifies that files not matched by any rule or +// heading pattern fall back to kind=research. +func TestProcess_Fallback(t *testing.T) { + root := testdataDir(t) + ing := newIngester() + + result, err := ing.Process(context.Background(), root) + if err != nil { + t.Fatalf("Process: %v", err) + } + + // notes/random.md has no frontmatter, no matching path rule, no + // classification headings. It should fall back to kind=research. + e, ok := findEntry(result.Entries, "Meeting notes") + if !ok { + t.Fatalf("expected entry 'Meeting notes'; entries: %v", titles(result.Entries)) + } + if e.Kind != lore.KindResearch { + t.Errorf("random notes kind: got %q, want %q", e.Kind, lore.KindResearch) + } +} + +// TestProcess_FileErrors verifies that an unreadable file produces a +// FileError in the result but does not abort the rest of the walk. +func TestProcess_FileErrors(t *testing.T) { + // Build a temp tree with one valid file and one unreadable file. + dir := t.TempDir() + goodPath := filepath.Join(dir, "good.md") + badPath := filepath.Join(dir, "bad.md") + + if err := os.WriteFile(goodPath, []byte("# Good\n\nContent.\n"), 0o644); err != nil { + t.Fatalf("write good.md: %v", err) + } + if err := os.WriteFile(badPath, []byte("# Bad\n\nContent.\n"), 0o000); err != nil { + t.Fatalf("write bad.md: %v", err) + } + + ing := newIngester() + result, err := ing.Process(context.Background(), dir) + if err != nil { + t.Fatalf("Process: %v", err) + } + + // good.md should produce one entry. + if len(result.Entries) == 0 { + t.Error("expected at least one entry from good.md") + } + + // bad.md should produce one FileError. + if len(result.Errors) == 0 { + t.Error("expected at least one FileError for unreadable bad.md") + } + + found := false + for _, fe := range result.Errors { + if fe.Path == badPath { + found = true + break + } + } + if !found { + t.Errorf("expected FileError for %s; got: %v", badPath, result.Errors) + } +} + +// TestWalker_SkipsHidden verifies that the walker does not descend into +// .git, node_modules, vendor, or other hidden directories. +func TestWalker_SkipsHidden(t *testing.T) { + dir := t.TempDir() + + // Create a file in a skipped directory. + for _, skip := range []string{".git", "node_modules", "vendor", ".hidden"} { + skipDir := filepath.Join(dir, skip) + if err := os.MkdirAll(skipDir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", skipDir, err) + } + f := filepath.Join(skipDir, "doc.md") + if err := os.WriteFile(f, []byte("# Should be skipped\n"), 0o644); err != nil { + t.Fatalf("write %s: %v", f, err) + } + } + + // Add one reachable file. + good := filepath.Join(dir, "readme.md") + if err := os.WriteFile(good, []byte("# Readme\n"), 0o644); err != nil { + t.Fatalf("write readme.md: %v", err) + } + + wr := ingest.WalkDir(dir, ingest.WalkerConfig{}) + if len(wr.Errors) != 0 { + t.Errorf("unexpected walk errors: %v", wr.Errors) + } + if len(wr.Paths) != 1 { + t.Errorf("expected exactly 1 path; got %d: %v", len(wr.Paths), wr.Paths) + } + if len(wr.Paths) == 1 && wr.Paths[0] != good { + t.Errorf("path: got %q, want %q", wr.Paths[0], good) + } +} + +// TestChunker_Headings verifies that H1/H2/H3 headings produce separate +// chunks, each with the correct title and source line number. +func TestChunker_Headings(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "doc.md") + + content := `# Title one + +First paragraph. + +## Section two + +Second paragraph. + +### Subsection three + +Third paragraph. +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write doc.md: %v", err) + } + + chunks, err := ingest.ChunkFile(path, dir) + if err != nil { + t.Fatalf("ChunkFile: %v", err) + } + + if len(chunks) != 3 { + t.Fatalf("expected 3 chunks, got %d: %v", len(chunks), chunkTitles(chunks)) + } + + wantTitles := []string{"Title one", "Section two", "Subsection three"} + for i, c := range chunks { + if c.Title != wantTitles[i] { + t.Errorf("chunk[%d].Title: got %q, want %q", i, c.Title, wantTitles[i]) + } + } + + // First chunk starts at line 1 (no frontmatter). + if chunks[0].Source != "doc.md:1" { + t.Errorf("chunk[0].Source: got %q, want %q", chunks[0].Source, "doc.md:1") + } +} + +// TestChunker_Frontmatter verifies that YAML front matter is parsed +// and attached to all chunks from the file. +func TestChunker_Frontmatter(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "fm.md") + + content := `--- +kind: decision +tags: [adr, test] +--- + +# My decision + +Body text. +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write fm.md: %v", err) + } + + chunks, err := ingest.ChunkFile(path, dir) + if err != nil { + t.Fatalf("ChunkFile: %v", err) + } + + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + + c := chunks[0] + if c.FM.Kind != "decision" { + t.Errorf("FM.Kind: got %q, want %q", c.FM.Kind, "decision") + } + if len(c.FM.Tags) != 2 || c.FM.Tags[0] != "adr" || c.FM.Tags[1] != "test" { + t.Errorf("FM.Tags: got %v, want [adr test]", c.FM.Tags) + } +} + +// TestChunker_NoHeadings verifies that a file with no headings returns a +// single chunk with the filename stem as the title. +func TestChunker_NoHeadings(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "plain.md") + + if err := os.WriteFile(path, []byte("Just some plain text.\n"), 0o644); err != nil { + t.Fatalf("write plain.md: %v", err) + } + + chunks, err := ingest.ChunkFile(path, dir) + if err != nil { + t.Fatalf("ChunkFile: %v", err) + } + + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + if chunks[0].Title != "plain" { + t.Errorf("Title: got %q, want %q", chunks[0].Title, "plain") + } +} + +// titles returns a slice of entry titles for diagnostic messages. +func titles(entries []lore.Entry) []string { + out := make([]string, len(entries)) + for i, e := range entries { + out[i] = e.Title + } + return out +} + +// chunkTitles returns a slice of chunk titles for diagnostic messages. +func chunkTitles(chunks []ingest.Chunk) []string { + out := make([]string, len(chunks)) + for i, c := range chunks { + out[i] = c.Title + } + return out +} diff --git a/pkg/lore/ingest/testdata/CLAUDE.md b/pkg/lore/ingest/testdata/CLAUDE.md new file mode 100644 index 0000000..bd4b0fe --- /dev/null +++ b/pkg/lore/ingest/testdata/CLAUDE.md @@ -0,0 +1,14 @@ +# lore agent bootstrap + +This file is loaded by MCP harnesses that auto-consume `.mcpb` configuration. +It carries repo-specific context for contributing agents. + +## First action + +Run `guild_session_start(project="lore")` before doing anything else. + +## Repo facts + +- Module: `github.com/mathomhaus/lore` +- Language: Go 1.23+ +- Test gate: `go test -race ./...` diff --git a/pkg/lore/ingest/testdata/docs/adr/0001-use-go.md b/pkg/lore/ingest/testdata/docs/adr/0001-use-go.md new file mode 100644 index 0000000..8853e7e --- /dev/null +++ b/pkg/lore/ingest/testdata/docs/adr/0001-use-go.md @@ -0,0 +1,28 @@ +--- +kind: decision +tags: [adr, language] +--- + +# Use Go for the implementation + +## Status + +Accepted + +## Context + +We needed a language for the lore library that compiles to a single binary, +has strong typing, good concurrency primitives, and a mature standard library. +Python and Node were considered but ruled out due to deployment complexity and +runtime overhead. + +## Decision + +Use Go 1.23+ as the implementation language for the lore library. + +## Consequences + +- Callers must use a Go toolchain to consume the library. +- Cross-language bindings (Python, JS) require a separate wrapper layer. +- The library ships as a Go module with no CGO requirements (pure-Go SQLite + via modernc.org/sqlite). diff --git a/pkg/lore/ingest/testdata/docs/concepts/auth.md b/pkg/lore/ingest/testdata/docs/concepts/auth.md new file mode 100644 index 0000000..a84a638 --- /dev/null +++ b/pkg/lore/ingest/testdata/docs/concepts/auth.md @@ -0,0 +1,22 @@ +# Authentication in lore + +## What is auth + +Authentication in the lore library controls which callers can read and write +entries. Lore itself is library-only and does not ship an HTTP server, so +authentication is the caller's responsibility. The library surface is +deliberately unaware of principals, sessions, or tokens. + +## Design choices + +Callers wrap the Store interface with their own authorization logic: + +```go +type authStore struct { + inner lore.Store + check func(ctx context.Context) error +} +``` + +This keeps the lore core free of auth policy and lets callers apply whatever +identity model their deployment needs (service account, OIDC, mTLS, etc.). diff --git a/pkg/lore/ingest/testdata/docs/runbooks/deploy.md b/pkg/lore/ingest/testdata/docs/runbooks/deploy.md new file mode 100644 index 0000000..aff7820 --- /dev/null +++ b/pkg/lore/ingest/testdata/docs/runbooks/deploy.md @@ -0,0 +1,26 @@ +# Deploy to production + +Runbook for deploying lore consumers to production. + +## Prerequisites + +- Go 1.23+ toolchain installed +- Access to the release repository +- Signing key available in the CI environment + +## Steps + +1. Tag the commit: `git tag v0.x.y` +2. Push the tag: `git push upstream v0.x.y` +3. Wait for the release workflow to complete. +4. Verify the release assets are present on the GitHub releases page. +5. Update the brew tap SHA if applicable. + +## Rollback + +If the release is broken, delete the tag and re-push after the fix: + +```bash +git tag -d v0.x.y +git push upstream :refs/tags/v0.x.y +``` diff --git a/pkg/lore/ingest/testdata/notes/random.md b/pkg/lore/ingest/testdata/notes/random.md new file mode 100644 index 0000000..ca956cd --- /dev/null +++ b/pkg/lore/ingest/testdata/notes/random.md @@ -0,0 +1,7 @@ +# Meeting notes 2026-03-14 + +Random notes from the weekly sync. Not categorized. + +- Discussed timeline for v0.1.1 release. +- Agreed to keep the public API minimal for the first stable release. +- Follow-up: open tracking tickets for the v0.2 backlog items. diff --git a/pkg/lore/ingest/walker.go b/pkg/lore/ingest/walker.go new file mode 100644 index 0000000..4bde39a --- /dev/null +++ b/pkg/lore/ingest/walker.go @@ -0,0 +1,159 @@ +package ingest + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const ( + // DefaultMaxFileSize is the default upper bound on file size. Files + // larger than this are skipped with a FileError. Override via + // WalkerConfig.MaxFileSize. + DefaultMaxFileSize int64 = 10 * 1024 * 1024 // 10 MB +) + +// WalkerConfig holds the knobs that govern the filesystem walk. +type WalkerConfig struct { + // MaxFileSize is the size threshold above which a file is skipped. + // Zero or negative means "use DefaultMaxFileSize". + MaxFileSize int64 +} + +// WalkResult is the output of a single WalkDir call. +type WalkResult struct { + // Paths contains the absolute paths of accepted files in walk order. + Paths []string + + // Errors collects per-entry failures: unreadable directories, oversized + // files, and similar conditions that do not abort the whole walk. + Errors []FileError +} + +// skipDirs is the set of directory basenames the walker never descends into. +var skipDirs = map[string]struct{}{ + ".git": {}, + "node_modules": {}, + "vendor": {}, +} + +// markdownExts is the set of file extensions the walker accepts. +// Comparison is case-insensitive so ".MD" is accepted too. +var markdownExts = map[string]struct{}{ + ".md": {}, + ".markdown": {}, +} + +// WalkDir walks root with filepath.WalkDir and returns the paths of Markdown +// files that pass the filter criteria. +// +// Walk semantics for v0.1.1: +// - One-shot, non-incremental. +// - Symlinks are NOT followed. +// - .gitignore patterns are NOT honored (v0.2 followup). +// +// Skipped unconditionally: +// - Directories named in skipDirs (.git, node_modules, vendor). +// - Hidden directories (basename starts with "."). +// - Non-Markdown files; only .md and .markdown extensions are accepted. +// - Files larger than cfg.MaxFileSize (default 10 MB). +func WalkDir(root string, cfg WalkerConfig) WalkResult { + maxSize := cfg.MaxFileSize + if maxSize <= 0 { + maxSize = DefaultMaxFileSize + } + + var res WalkResult + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // Unreadable entry: record and continue. + res.Errors = append(res.Errors, FileError{Path: path, Err: err}) + if d != nil && d.IsDir() { + return filepath.SkipDir + } + return nil + } + + base := d.Name() + + if d.IsDir() { + if path == root { + return nil + } + if _, skip := skipDirs[base]; skip { + return filepath.SkipDir + } + if strings.HasPrefix(base, ".") { + return filepath.SkipDir + } + return nil + } + + // Only regular files; skip symlinks, pipes, devices, etc. + if d.Type() != 0 { + return nil + } + + // Extension filter (case-insensitive). + ext := strings.ToLower(filepath.Ext(base)) + if _, ok := markdownExts[ext]; !ok { + return nil + } + + // Size check. + info, err := os.Stat(path) + if err != nil { + res.Errors = append(res.Errors, FileError{Path: path, Err: err}) + return nil + } + if info.Size() > maxSize { + res.Errors = append(res.Errors, FileError{ + Path: path, + Err: newFileTooLargeError(path, info.Size(), maxSize), + }) + return nil + } + + res.Paths = append(res.Paths, path) + return nil + }) + + if err != nil { + res.Errors = append(res.Errors, FileError{Path: root, Err: err}) + } + + return res +} + +func newFileTooLargeError(path string, size, limit int64) error { + return &fileTooLargeError{path: path, size: size, limit: limit} +} + +type fileTooLargeError struct { + path string + size int64 + limit int64 +} + +func (e *fileTooLargeError) Error() string { + return fmt.Sprintf("ingest: walker: file too large: %s (%s > %s)", + filepath.Base(e.path), formatBytes(e.size), formatBytes(e.limit)) +} + +func formatBytes(n int64) string { + const ( + kb = 1024 + mb = 1024 * kb + ) + switch { + case n >= mb: + return fmt.Sprintf("%.1f MB", float64(n)/float64(mb)) + case n >= kb: + return fmt.Sprintf("%.1f KB", float64(n)/float64(kb)) + default: + return fmt.Sprintf("%d B", n) + } +}