[lore 6/7] Ingester + heuristic classifier (Path B)

README.md

@@ -99,6 +99,80 @@ The `store.Store` interface is backend-agnostic. Swap `sqlite.New` for any
 implementation that satisfies the interface to use a different storage engine
 without changing callers.
 
+### Path B: document ingestion
+
+Path B ingests existing Markdown document trees into lore entries. The
+ingester is a pure functional transform: it returns entries and the caller
+writes them to a Store.
+
+```go
+import (
+    "context"
+    "log"
+
+    "github.com/mathomhaus/lore/pkg/lore/ingest/heuristic"
+)
+
+func main() {
+    ing := heuristic.NewIngester()
+
+    result, err := ing.Process(context.Background(), "/workspace/docs")
+    if err != nil {
+        log.Fatal(err)
+    }
+
+    for _, fe := range result.Errors {
+        log.Printf("warn: %v", fe)
+    }
+
+    for _, e := range result.Entries {
+        log.Printf("entry kind=%s title=%q source=%s", e.Kind, e.Title, e.Source)
+        // write e to your store here
+    }
+}
+```
+
+#### Classification priority
+
+The heuristic ingester classifies each chunk using this priority order (first
+match wins):
+
+1. YAML front matter with an explicit `kind:` field.
+2. Path rules: `docs/adr/*.md` maps to decision+adr; `docs/runbooks/*.md`
+   maps to procedure+runbook; `CLAUDE.md`/`agents.md`/`skills.md` map to
+   reference+agent-config; and so on. See `heuristic.DefaultRules()`.
+3. Heading keywords: `## What is` maps to explanation; `## Decision` /
+   `## Context` / `## Consequences` maps to decision; `## Procedure` /
+   `## Steps` maps to procedure; etc.
+4. Fallback: kind=research (catch-all).
+
+#### Customizing rules
+
+```go
+import "github.com/mathomhaus/lore/pkg/lore/ingest/heuristic"
+
+rules := heuristic.DefaultRules()
+rules = append(rules, heuristic.Rule{
+    PathGlob: "docs/specs/*.md",
+    Kind:     lore.KindDecision,
+    Tags:     []string{"spec"},
+})
+
+ing := heuristic.NewIngester(
+    heuristic.WithRules(rules),
+    heuristic.WithLogger(slog.Default()),
+)
+```
+
+#### Walker behavior (v0.1.1)
+
+- Only `.md` and `.markdown` files are processed.
+- `.git/`, `node_modules/`, `vendor/`, and any hidden directory (name
+  starting with `.`) are skipped unconditionally.
+- Files larger than 10 MB are skipped with a FileError.
+- Symlinks are not followed.
+- `.gitignore` patterns are not honored (planned for v0.2).
+
 ## Status: pre-v1.0
 
 Lore is pre-v1.0. The exported surface is stable in shape but may change in

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2
 	go.opentelemetry.io/otel v1.43.0
 	go.opentelemetry.io/otel/trace v1.43.0
+	gopkg.in/yaml.v3 v3.0.1
 	modernc.org/sqlite v1.50.0
 )
 

go.sum

@@ -19,6 +19,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -27,6 +31,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2 h1:jW34+ipjoob+BfuLfALS6FBJh84KJcmzFowmxsImA/o=
 github.com/shota3506/onnxruntime-purego v0.0.0-20260315223538-8db8bd7424b2/go.mod h1:jsOuI9QymOruQ9UdLrCbMPFykhI13d5FcuV60R1wJ6Q=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
@@ -48,6 +54,9 @@ golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
 golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
 golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=

pkg/lore/ingest/chunker.go

@@ -0,0 +1,209 @@
+package ingest
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Frontmatter holds the fields we parse from a YAML front matter block.
+// All fields are optional; unrecognized keys are silently ignored.
+type Frontmatter struct {
+	Kind string   `yaml:"kind"`
+	Tags []string `yaml:"tags"`
+}
+
+// Chunk is an intermediate representation of one heading-bounded section of
+// a Markdown file. It is enriched into a lore.Entry by the classifier.
+type Chunk struct {
+	// Title is the text of the heading that opened this chunk, or the file
+	// stem when the file has no headings.
+	Title string
+
+	// Body is the raw Markdown for this chunk, including the opening heading
+	// line.
+	Body string
+
+	// Source is "<repo-relative-path>:<line-number>" of the heading.
+	Source string
+
+	// FM is the parsed front matter from the file. It is identical for
+	// every chunk derived from the same file; per-chunk overrides are not
+	// supported in v0.1.1.
+	FM Frontmatter
+
+	// FilePath is the absolute path of the source file.
+	FilePath string
+}
+
+// ChunkFile reads the Markdown file at path, extracts any YAML front matter,
+// and splits the remainder at H1/H2/H3 boundaries. Each boundary produces one
+// Chunk. If the file has no headings, the whole body is returned as a single
+// Chunk whose title is the filename stem.
+//
+// The Source field on each chunk is set to "<relPath>:<lineNumber>" where
+// relPath is path relative to root. If root is empty or path is not under
+// root, the base name is used instead.
+func ChunkFile(path, root string) ([]Chunk, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("ingest: chunker: open %s: %w", path, err)
+	}
+	defer func() { _ = f.Close() }()
+
+	relPath := RelativeOrBase(path, root)
+
+	scanner := bufio.NewScanner(f)
+
+	var fm Frontmatter
+	var bodyLines []string
+
+	lineNum := 0
+	inFrontmatter := false
+	var fmLines []string
+	fmDone := false
+	bodyLineOffset := 0
+
+	for scanner.Scan() {
+		lineNum++
+		line := scanner.Text()
+
+		if lineNum == 1 && line == "---" {
+			inFrontmatter = true
+			continue
+		}
+
+		if inFrontmatter {
+			if line == "---" || line == "..." {
+				inFrontmatter = false
+				fmDone = true
+				bodyLineOffset = lineNum
+				continue
+			}
+			fmLines = append(fmLines, line)
+			continue
+		}
+
+		if !fmDone {
+			bodyLineOffset = lineNum - 1
+			fmDone = true
+		}
+
+		bodyLines = append(bodyLines, line)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("ingest: chunker: scan %s: %w", path, err)
+	}
+
+	if len(fmLines) > 0 {
+		_ = yaml.Unmarshal([]byte(strings.Join(fmLines, "\n")), &fm)
+	}
+
+	// Absolute line number of the first body line.
+	firstBodyLine := bodyLineOffset + 1
+
+	// Find all H1/H2/H3 boundary positions.
+	type boundary struct {
+		lineIdx int
+		title   string
+	}
+
+	var boundaries []boundary
+	for i, line := range bodyLines {
+		if title, ok := ExtractHeading(line); ok {
+			boundaries = append(boundaries, boundary{lineIdx: i, title: title})
+		}
+	}
+
+	fileTitle := stemName(filepath.Base(path))
+
+	// No headings: whole body is one chunk.
+	if len(boundaries) == 0 {
+		body := strings.Join(bodyLines, "\n")
+		src := fmt.Sprintf("%s:%d", relPath, firstBodyLine)
+		if strings.TrimSpace(body) == "" {
+			return nil, nil
+		}
+		return []Chunk{{
+			Title:    fileTitle,
+			Body:     body,
+			Source:   src,
+			FM:       fm,
+			FilePath: path,
+		}}, nil
+	}
+
+	chunks := make([]Chunk, 0, len(boundaries))
+	for idx, b := range boundaries {
+		var endIdx int
+		if idx+1 < len(boundaries) {
+			endIdx = boundaries[idx+1].lineIdx
+		} else {
+			endIdx = len(bodyLines)
+		}
+
+		chunkBodyLines := bodyLines[b.lineIdx:endIdx]
+		body := strings.Join(chunkBodyLines, "\n")
+		absLineNum := firstBodyLine + b.lineIdx
+		src := fmt.Sprintf("%s:%d", relPath, absLineNum)
+
+		chunks = append(chunks, Chunk{
+			Title:    b.title,
+			Body:     body,
+			Source:   src,
+			FM:       fm,
+			FilePath: path,
+		})
+	}
+
+	return chunks, nil
+}
+
+// ExtractHeading returns the heading text and true if line is an ATX-style
+// H1, H2, or H3 Markdown heading (one, two, or three leading # characters
+// followed by a space and non-empty text). Returns ("", false) for anything
+// else.
+func ExtractHeading(line string) (string, bool) {
+	if len(line) < 3 || line[0] != '#' {
+		return "", false
+	}
+	level := 0
+	for level < len(line) && line[level] == '#' {
+		level++
+	}
+	if level > 3 || level >= len(line) || line[level] != ' ' {
+		return "", false
+	}
+	title := strings.TrimSpace(line[level+1:])
+	if title == "" {
+		return "", false
+	}
+	return title, true
+}
+
+// stemName returns the filename without its extension.
+func stemName(base string) string {
+	ext := filepath.Ext(base)
+	if ext == "" {
+		return base
+	}
+	return base[:len(base)-len(ext)]
+}
+
+// RelativeOrBase returns path relative to root. If the path is not under root
+// or Rel returns an error, it falls back to filepath.Base(path).
+func RelativeOrBase(path, root string) string {
+	if root == "" {
+		return filepath.Base(path)
+	}
+	rel, err := filepath.Rel(root, path)
+	if err != nil || strings.HasPrefix(rel, "..") {
+		return filepath.Base(path)
+	}
+	return rel
+}