From 46f1c700fcec8b240b001069a3637a9390410b7a Mon Sep 17 00:00:00 2001 From: Kunal Lanjewar <5488221+kunallanjewar@users.noreply.github.com> Date: Tue, 28 Apr 2026 07:55:40 -0700 Subject: [PATCH] feat: repo scaffold and core types for lore v0.1.1 Bootstraps the mathomhaus/lore Go module with the type surface every later quest in the v0.1.1 series builds on: - go.mod at module path github.com/mathomhaus/lore, Go 1.23 minimum, no external dependencies yet (interfaces and reference impls land in subsequent quests). - pkg/lore/doc.go: package doc explaining library scope (no CLI, no MCP server, no service) and the three pluggable interfaces (Store, Embedder, VectorStore) that compose a Retriever and an optional Ingester. - pkg/lore/kind.go: Kind typed string with the eight canonical constants (decision, principle, procedure, reference, explanation, observation, research, idea), AllKinds, and Validate that wraps ErrInvalidKind. - pkg/lore/errors.go: sentinel errors (ErrNotFound, ErrDuplicate, ErrInvalidKind, ErrInvalidArgument, ErrConflict, ErrUnsupported, ErrClosed) intended to be wrapped with fmt.Errorf %w. - pkg/lore/lore.go: Entry, Edge, SearchHit, ListOpts, SearchOpts. Each field documents nullable and slice semantics so Store implementations can persist them consistently. - README.md: positioning paragraph, install hint, pre-v1.0 stability disclaimer, attribution to mathomhaus/guild, link to the spec doc. - .gitignore tuned for Go projects. - .github/workflows/ci.yml: matrix on go 1.23 and 1.24 across Ubuntu and macOS, runs go vet, go build, and go test -race -count=1. Tests cover Kind validation (canonical and rejected inputs), AllKinds size and membership, sentinel-error distinctness and wrap-preservation, and zero-value sanity for the public structs. --- .github/workflows/ci.yml | 41 ++++++++++++ .gitignore | 37 +++++++++++ README.md | 56 +++++++++++++++- go.mod | 3 + pkg/lore/doc.go | 56 ++++++++++++++++ pkg/lore/errors.go | 40 ++++++++++++ pkg/lore/errors_test.go | 41 ++++++++++++ pkg/lore/kind.go | 91 ++++++++++++++++++++++++++ pkg/lore/kind_test.go | 78 ++++++++++++++++++++++ pkg/lore/lore.go | 136 +++++++++++++++++++++++++++++++++++++++ pkg/lore/lore_test.go | 39 +++++++++++ 11 files changed, 617 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 pkg/lore/doc.go create mode 100644 pkg/lore/errors.go create mode 100644 pkg/lore/errors_test.go create mode 100644 pkg/lore/kind.go create mode 100644 pkg/lore/kind_test.go create mode 100644 pkg/lore/lore.go create mode 100644 pkg/lore/lore_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..dc6a035 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + test: + name: test (go ${{ matrix.go }} / ${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + go: ["1.23", "1.24"] + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: setup go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + + - name: go mod download + run: go mod download + + - name: go vet + run: go vet ./... + + - name: go build + run: go build ./... + + - name: go test + run: go test -race -count=1 ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4567cb6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Binaries +*.exe +*.exe~ +*.dll +*.so +*.dylib +/bin/ +/dist/ + +# Test, coverage, profiling +*.test +*.out +*.prof +coverage.txt +coverage.html +coverage.out + +# Go workspace and toolchain caches +go.work +go.work.sum +.go-cache/ + +# Editor and IDE state +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store + +# Environment / local config +.env +.env.local + +# Local scratch +/tmp/ +/scratch/ diff --git a/README.md b/README.md index ce266d2..7256208 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,56 @@ # lore -Structured knowledge primitive for AI agents. Apache 2 OSS Go library. Pluggable storage, embedding, vector store. BYO ingestion, serving, UI. + +A structured knowledge primitive for AI agents. Apache 2.0 OSS Go library. + +Lore stores classified knowledge entries (decisions, principles, procedures, +references, explanations, observations, research, ideas) and the typed edges +that connect them, then serves them back to retrieval pipelines that combine +lexical and semantic ranking. It ships as a Go library, not a service: callers +compose it into their own MCP servers, HTTP services, ingestion pipelines, or +CLI tools. + +The library is built around three pluggable interfaces (`Store`, `Embedder`, +`VectorStore`) plus a composing `Retriever` and an optional `Ingester`. Each +interface ships with an in-process reference implementation (modernc.org/sqlite, +BGE int8, sqlite-vec) so a single binary can run against a local SQLite file +out of the box. Swap any of the three for Postgres, a remote embedding API, +pgvector, or anything else by implementing the interface. + +## Install + +``` +go get github.com/mathomhaus/lore@latest +``` + +Requires Go 1.23 or newer. + +## Status: pre-v1.0 + +Lore is pre-v1.0. The exported surface is stable in shape but may change in +detail between minor versions. Pin to a version, read release notes before +upgrading, and expect occasional breakage on `main`. + +## What lore is not + +- Not a CLI binary. Not an MCP server. Not an HTTP server. Not a UI. +- Not a hosted service. Not multi-tenant. Not an LLM client. +- Not a replacement for a full retrieval-augmented-generation framework. + +Lore is the substrate. Everything above is a consumer's choice. + +## Attribution + +Lore extracts and generalizes the storage, embedding, and retrieval primitives +originally built inside [`mathomhaus/guild`](https://github.com/mathomhaus/guild). +Guild remains the opinionated agent-coordination platform that adds +quest, oath, and brief on top of these primitives. + +## Spec + +The architectural rationale and product positioning that informs this library +lives in the maintainer's notes at +`~/Library/CloudStorage/SynologyDrive-Obsidian/Personal/01 Projects/Agent Guild/Positioning/lore-product-mvp-2026-04-27.md`. + +## License + +Apache License 2.0. See [LICENSE](./LICENSE). diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..797656a --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/mathomhaus/lore + +go 1.23 diff --git a/pkg/lore/doc.go b/pkg/lore/doc.go new file mode 100644 index 0000000..68bdb52 --- /dev/null +++ b/pkg/lore/doc.go @@ -0,0 +1,56 @@ +// Package lore is a structured knowledge primitive for AI agents. +// +// Lore stores classified knowledge entries (decisions, principles, procedures, +// references, explanations, observations, research, ideas) and the edges that +// connect them, then serves them back to retrieval pipelines that combine +// lexical and semantic ranking. It ships as a Go library, not a service: +// callers compose it into their own MCP servers, HTTP services, ingestion +// pipelines, or CLI tools. +// +// # Scope +// +// Lore v0.1.1 is library-only. It does not ship a CLI binary, an MCP server, +// an HTTP server, or a UI. It does not embed an LLM or talk to a remote +// service. It is the substrate; consumers compose the rest. +// +// # The three pluggable interfaces +// +// Lore is built around three swap points so users can mix in-process reference +// implementations with their own backends without rewriting retrieval logic: +// +// - Store persists entries and edges. The reference implementation lives in +// pkg/lore/store/sqlite and uses modernc.org/sqlite (pure Go) plus FTS5 for +// lexical search. Replace it with Postgres, MySQL, or any other engine by +// implementing the interface in pkg/lore/store. +// +// - Embedder turns text into vectors. The reference implementation lives in +// pkg/lore/embed/bge and runs an int8-quantized BGE model in process. +// Replace it with a remote embedding API or a different local model by +// implementing the interface in pkg/lore/embed. +// +// - VectorStore persists and queries vectors. The reference implementation +// lives in pkg/lore/vector/sqlitevec and uses the sqlite-vec extension. +// Replace it with pgvector, Qdrant, Weaviate, or any other engine by +// implementing the interface in pkg/lore/vector. +// +// On top of these three, lore composes a Retriever that runs lexical and +// vector queries in parallel and fuses results with reciprocal-rank fusion, +// and an optional Ingester that walks document trees and classifies chunks +// into entries on Path B (document ingestion). Path A (agent inscribe) goes +// straight through Store and Embedder without an ingest pipeline. +// +// # Caller-owned dependencies +// +// Constructors in this library accept already-initialized resources: +// *sql.DB, *http.Client, *slog.Logger, OpenTelemetry providers, and so on. +// The library does not open database connections, parse URLs, or read +// environment variables on the caller's behalf. This keeps the library +// stateless beyond its injected dependencies and safe to deploy across +// multiple replicas. +// +// # Stability +// +// Lore is pre-v1.0. The exported surface is stable in shape but may change in +// detail between minor versions. Pin to a version and read release notes +// before upgrading. +package lore diff --git a/pkg/lore/errors.go b/pkg/lore/errors.go new file mode 100644 index 0000000..d8bc88c --- /dev/null +++ b/pkg/lore/errors.go @@ -0,0 +1,40 @@ +package lore + +import "errors" + +// Sentinel errors are the canonical machine-readable failure modes. Callers +// match them with errors.Is. Implementations that wrap these with +// fmt.Errorf("...: %w", err) preserve the chain. +var ( + // ErrNotFound indicates a requested entry, edge, or row does not exist. + ErrNotFound = errors.New("lore: not found") + + // ErrDuplicate indicates a write would create a duplicate of an existing + // row keyed by a uniqueness constraint (for example same-title same-kind + // inside a single project, depending on the Store implementation). + ErrDuplicate = errors.New("lore: duplicate") + + // ErrInvalidKind indicates a Kind value is outside the canonical + // taxonomy. Returned by Kind.Validate and by write paths that classify + // entries. + ErrInvalidKind = errors.New("lore: invalid kind") + + // ErrInvalidArgument indicates a caller-supplied input failed validation + // (empty title, malformed source, negative limit, and so on). Callers + // should fix the input rather than retry. + ErrInvalidArgument = errors.New("lore: invalid argument") + + // ErrConflict indicates an optimistic-concurrency check failed: the + // underlying row changed between read and write. Callers may retry after + // re-reading current state. + ErrConflict = errors.New("lore: conflict") + + // ErrUnsupported indicates the requested operation is not implemented by + // the active backend (for example a vector query against a Store that + // only implements lexical retrieval). + ErrUnsupported = errors.New("lore: unsupported") + + // ErrClosed indicates the component has been closed and rejects further + // I/O. Callers must construct a new instance to continue. + ErrClosed = errors.New("lore: closed") +) diff --git a/pkg/lore/errors_test.go b/pkg/lore/errors_test.go new file mode 100644 index 0000000..d1bd630 --- /dev/null +++ b/pkg/lore/errors_test.go @@ -0,0 +1,41 @@ +package lore + +import ( + "errors" + "fmt" + "testing" +) + +// Sentinel errors must remain distinct so callers can pattern-match on them. +func TestSentinelErrors_Distinct(t *testing.T) { + all := []error{ + ErrNotFound, + ErrDuplicate, + ErrInvalidKind, + ErrInvalidArgument, + ErrConflict, + ErrUnsupported, + ErrClosed, + } + for i, a := range all { + for j, b := range all { + if i == j { + continue + } + if errors.Is(a, b) { + t.Errorf("sentinels %d and %d collide: %v == %v", i, j, a, b) + } + } + } +} + +// Wrapping with fmt.Errorf("%w") must preserve errors.Is matching. +func TestSentinelErrors_Wrap(t *testing.T) { + wrapped := fmt.Errorf("read entry 42: %w", ErrNotFound) + if !errors.Is(wrapped, ErrNotFound) { + t.Fatalf("wrapped ErrNotFound did not match via errors.Is") + } + if errors.Is(wrapped, ErrDuplicate) { + t.Fatalf("wrapped ErrNotFound matched unrelated sentinel ErrDuplicate") + } +} diff --git a/pkg/lore/kind.go b/pkg/lore/kind.go new file mode 100644 index 0000000..d03468e --- /dev/null +++ b/pkg/lore/kind.go @@ -0,0 +1,91 @@ +package lore + +import "fmt" + +// Kind classifies a lore entry. The canonical taxonomy has eight values that +// together cover both document-derived knowledge (procedure, reference, +// explanation) and agent-derived knowledge (decision, principle, observation, +// research, idea). Diátaxis prior art motivates the document-derived three; +// the rest carry classifications agents naturally produce while reasoning. +// +// Callers should treat Kind as an opaque typed string and use the constants +// declared below. New kinds may be added in future versions; consumers that +// switch on Kind should always handle an unknown value gracefully. +type Kind string + +// Canonical kinds. Lore validates these on every write path and rejects any +// other value with ErrInvalidKind. +const ( + // KindDecision records a choice made with rationale. ADRs, "we chose X + // because Y", policy decisions. + KindDecision Kind = "decision" + + // KindPrinciple records a durable rule or invariant. "Always X", "never Y", + // coding standards. + KindPrinciple Kind = "principle" + + // KindProcedure records a step-by-step how-to. Runbooks, deploy guides, + // incident response playbooks. + KindProcedure Kind = "procedure" + + // KindReference records a fact to look up. Service catalogs, API specs, + // configuration tables, glossaries. + KindReference Kind = "reference" + + // KindExplanation records a concept or mental model. Architecture + // overviews, "how auth works", design walkthroughs. + KindExplanation Kind = "explanation" + + // KindObservation records an empirical finding. "We measured X", "we saw + // Y", postmortems. + KindObservation Kind = "observation" + + // KindResearch records the result of an investigation. Spike outputs, + // "we explored X, here's what we found". + KindResearch Kind = "research" + + // KindIdea records a proposal not yet decided. Design sketches, open + // questions, "what if we did X". + KindIdea Kind = "idea" +) + +// AllKinds returns the canonical kinds in display order. The order is stable +// and may be relied upon by UIs, tests, and migration tooling. +func AllKinds() []Kind { + return []Kind{ + KindDecision, + KindPrinciple, + KindProcedure, + KindReference, + KindExplanation, + KindObservation, + KindResearch, + KindIdea, + } +} + +// Validate reports whether k is one of the canonical kinds. It returns +// ErrInvalidKind wrapped with the offending value when validation fails so +// callers can inspect both the sentinel and the bad input. +func (k Kind) Validate() error { + switch k { + case KindDecision, + KindPrinciple, + KindProcedure, + KindReference, + KindExplanation, + KindObservation, + KindResearch, + KindIdea: + return nil + default: + return fmt.Errorf("kind %q: %w", string(k), ErrInvalidKind) + } +} + +// String returns the wire-format string for k. It is the identity function on +// the underlying string and is provided for symmetry with fmt.Stringer +// expectations. +func (k Kind) String() string { + return string(k) +} diff --git a/pkg/lore/kind_test.go b/pkg/lore/kind_test.go new file mode 100644 index 0000000..2dd66e6 --- /dev/null +++ b/pkg/lore/kind_test.go @@ -0,0 +1,78 @@ +package lore + +import ( + "errors" + "testing" +) + +func TestKindValidate_Canonical(t *testing.T) { + for _, k := range AllKinds() { + t.Run(string(k), func(t *testing.T) { + if err := k.Validate(); err != nil { + t.Fatalf("canonical kind %q rejected: %v", k, err) + } + }) + } +} + +func TestKindValidate_Invalid(t *testing.T) { + cases := []Kind{ + "", + "unknown", + "Decision", // case-sensitive + "decisions", // plural + " decision", + } + for _, k := range cases { + t.Run(string(k), func(t *testing.T) { + err := k.Validate() + if err == nil { + t.Fatalf("expected error for %q, got nil", k) + } + if !errors.Is(err, ErrInvalidKind) { + t.Fatalf("expected ErrInvalidKind, got %v", err) + } + }) + } +} + +func TestAllKinds_Count(t *testing.T) { + got := AllKinds() + const want = 8 + if len(got) != want { + t.Fatalf("AllKinds returned %d kinds, want %d", len(got), want) + } +} + +func TestAllKinds_Membership(t *testing.T) { + want := map[Kind]bool{ + KindDecision: true, + KindPrinciple: true, + KindProcedure: true, + KindReference: true, + KindExplanation: true, + KindObservation: true, + KindResearch: true, + KindIdea: true, + } + got := AllKinds() + if len(got) != len(want) { + t.Fatalf("AllKinds size mismatch: got %d, want %d", len(got), len(want)) + } + seen := make(map[Kind]bool, len(got)) + for _, k := range got { + if !want[k] { + t.Errorf("AllKinds returned unexpected kind %q", k) + } + if seen[k] { + t.Errorf("AllKinds returned duplicate kind %q", k) + } + seen[k] = true + } +} + +func TestKindString(t *testing.T) { + if got := KindDecision.String(); got != "decision" { + t.Fatalf("KindDecision.String() = %q, want %q", got, "decision") + } +} diff --git a/pkg/lore/lore.go b/pkg/lore/lore.go new file mode 100644 index 0000000..5ae0b5b --- /dev/null +++ b/pkg/lore/lore.go @@ -0,0 +1,136 @@ +package lore + +import "time" + +// Entry is the canonical record stored by lore. One entry corresponds to one +// classified piece of knowledge. The shape is deliberately small: the eight +// canonical kinds plus open-ended tags and a free-form metadata map cover +// both document-derived ingestion and agent-mediated inscribe paths without +// forcing schema churn for new use cases. +// +// Implementations of Store decide how to persist nullable and slice fields +// (typically: empty string for unset Source, empty slice for no Tags, nil +// map for no Metadata). The library treats nil and empty as equivalent. +type Entry struct { + // ID is the storage-assigned identifier. Zero on entries that have not + // yet been persisted. + ID int64 + + // Project scopes an entry to a logical workspace. Implementations may + // use this for filtering, partitioning, or access control. Empty string + // is permitted and represents the default project. + Project string + + // Kind is one of the canonical eight values declared in kind.go. + Kind Kind + + // Title is a short distinctive label, suitable for display in lists and + // for boosting in lexical retrieval. Required on write. + Title string + + // Body is the full prose content of the entry. May be markdown, plain + // text, or any other UTF-8 payload the caller wishes to store. + Body string + + // Source records where the entry came from. Free-form by design: + // implementations and consumers may agree on conventions like + // "github://owner/repo/path.md#anchor" or "agent://session/inscribe-id". + Source string + + // Tags are open-ended classifiers. They complement Kind for finer + // specificity (a decision tagged "adr,architecture,microservices") and + // are searchable by lexical retrievers. + Tags []string + + // Metadata carries arbitrary string key/value pairs for extensibility. + // Reserved keys may be defined by future versions; consumers should + // namespace their own keys to avoid collisions. + Metadata map[string]string + + // CreatedAt is the wall-clock time the entry was first persisted. + CreatedAt time.Time + + // UpdatedAt is the wall-clock time the entry was last modified. + UpdatedAt time.Time +} + +// Edge is a typed directed link between two entries. Edges enable provenance +// tracing ("entry A informs entry B"), supersession chains, and arbitrary +// caller-defined relations. Lore does not enforce a closed vocabulary on +// Relation: implementations and consumers pick their own conventions. +type Edge struct { + // FromID and ToID are the storage IDs of the linked entries. + FromID int64 + ToID int64 + + // Relation labels the edge. Common values include "informs", + // "supersedes", "contradicts", "depends-on", and "describes". The + // library does not validate the value; consumers may. + Relation string + + // Weight optionally orders edges of the same relation between the same + // pair. Implementations may use it for ranking; zero is a valid neutral. + Weight float64 + + // CreatedAt is the wall-clock time the edge was first persisted. + CreatedAt time.Time +} + +// SearchHit is one ranked result from a retrieval call. The Score field is +// implementation-defined: lexical retrievers typically return BM25 or TF-IDF +// scores, vector retrievers return similarity (often cosine), and fused +// retrievers return reciprocal-rank-fusion scores. Higher is always better. +type SearchHit struct { + // Entry is the matched record. + Entry Entry + + // Score is the retrieval score for this hit. Comparable across hits + // from the same query call; not necessarily comparable across queries + // or across retriever implementations. + Score float64 + + // Highlights optionally carries snippets from the matched body, with + // implementation-defined markers around the matching terms. May be nil. + Highlights []string +} + +// ListOpts narrows a list query. Zero values mean "no filter". +type ListOpts struct { + // Project, when non-empty, restricts results to a single project. + Project string + + // Kind, when non-empty, restricts results to a single kind. + Kind Kind + + // Tag, when non-empty, restricts results to entries that carry this tag. + Tag string + + // Limit caps the number of returned entries. Zero means + // implementation-default (typically a small page size); negative is + // invalid and returns ErrInvalidArgument. + Limit int + + // Offset skips the first N results. Zero means "from the start". + // Negative is invalid and returns ErrInvalidArgument. + Offset int +} + +// SearchOpts configures a retrieval call. Zero values mean +// "implementation-default". +type SearchOpts struct { + // Project, when non-empty, restricts results to a single project. + Project string + + // Kinds, when non-empty, restricts results to entries whose Kind is + // in the slice. + Kinds []Kind + + // Tags, when non-empty, restricts results to entries that carry every + // listed tag (intersection, not union). + Tags []string + + // Limit caps the number of returned hits. Zero means + // implementation-default; negative is invalid and returns + // ErrInvalidArgument. + Limit int +} diff --git a/pkg/lore/lore_test.go b/pkg/lore/lore_test.go new file mode 100644 index 0000000..f2b3ebb --- /dev/null +++ b/pkg/lore/lore_test.go @@ -0,0 +1,39 @@ +package lore + +import "testing" + +// Compile-time sanity: zero values of the public structs are usable. +func TestEntry_ZeroValue(t *testing.T) { + var e Entry + if e.ID != 0 || e.Title != "" || e.Tags != nil || e.Metadata != nil { + t.Fatalf("Entry zero value has unexpected non-zero fields: %+v", e) + } +} + +func TestEdge_ZeroValue(t *testing.T) { + var ed Edge + if ed.FromID != 0 || ed.ToID != 0 || ed.Relation != "" || ed.Weight != 0 { + t.Fatalf("Edge zero value has unexpected non-zero fields: %+v", ed) + } +} + +func TestSearchHit_ZeroValue(t *testing.T) { + var h SearchHit + if h.Score != 0 || h.Highlights != nil { + t.Fatalf("SearchHit zero value has unexpected non-zero fields: %+v", h) + } +} + +func TestListOpts_ZeroValue(t *testing.T) { + var o ListOpts + if o.Project != "" || o.Kind != "" || o.Tag != "" || o.Limit != 0 || o.Offset != 0 { + t.Fatalf("ListOpts zero value has unexpected non-zero fields: %+v", o) + } +} + +func TestSearchOpts_ZeroValue(t *testing.T) { + var o SearchOpts + if o.Project != "" || o.Kinds != nil || o.Tags != nil || o.Limit != 0 { + t.Fatalf("SearchOpts zero value has unexpected non-zero fields: %+v", o) + } +}