From eb1e1a5666a4266aeb4ec3d30b14ab9275eb6a23 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 01:15:04 -0400 Subject: [PATCH 1/8] feat(sdk): add v2 local Go SDK core (middleware yet to be added) This lands the Phase 1 Slice 2 foundation on top of the v2.0 schema package: - add Init, Shutdown, Stats, From, Explain, Step, StepVoid, Fail, NewError, and Suppress - emit final v2.0 wide events to a writer in local mode - add request-scoped buffering with MaxSteps, MaxLogs, and MaxBufferBytes enforcement - degrade oversized requests to header-only output instead of breaking user flows - synthesize anchors from the first observable failing step - reject reserved WAYLOG_* lifecycle codes and count misuse in stats - support suppressed requests as header-only final events - keep SetField snapshot semantics via deep clone while preserving shallow per-log field merging on the hot path - add local explainability from the in-memory request buffer - add tests covering lifecycle behavior, caps, suppression, reserved-code handling, redaction, and concurrency This slice intentionally does not include HTTP middleware, ingest transport, dev dual-emit, or framework adapters. --- pkg/waylog/v2/assemble.go | 159 ++++++++++ pkg/waylog/v2/errors.go | 72 +++++ pkg/waylog/v2/explain.go | 133 ++++++++ pkg/waylog/v2/logger.go | 106 +++++++ pkg/waylog/v2/request.go | 409 ++++++++++++++++++++++++ pkg/waylog/v2/sdk_test.go | 651 ++++++++++++++++++++++++++++++++++++++ pkg/waylog/v2/step.go | 149 +++++++++ pkg/waylog/v2/waylog.go | 203 ++++++++++++ 8 files changed, 1882 insertions(+) create mode 100644 pkg/waylog/v2/assemble.go create mode 100644 pkg/waylog/v2/errors.go create mode 100644 pkg/waylog/v2/explain.go create mode 100644 pkg/waylog/v2/logger.go create mode 100644 pkg/waylog/v2/request.go create mode 100644 pkg/waylog/v2/sdk_test.go create mode 100644 pkg/waylog/v2/step.go create mode 100644 pkg/waylog/v2/waylog.go diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go new file mode 100644 index 0000000..e5f0d81 --- /dev/null +++ b/pkg/waylog/v2/assemble.go @@ -0,0 +1,159 @@ +package waylogv2 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +// Finalize seals the request, assembles the v2.0 wide event, runs the +// optional Redactor, writes the JSON line to Output, and removes the request +// from the active set. Subsequent calls on the same context return (nil, nil) +// and increment LateCompletionAfterEmit. +// +// Intended for middleware and adapter authors. +func Finalize(ctx context.Context) (*eventv2.Event, error) { + r := requestFromContext(ctx) + if r == nil { + return nil, ErrNoActiveRequest + } + + r.mu.Lock() + if r.sealed { + r.mu.Unlock() + r.sdk.lateAfterEmit.Add(1) + return nil, nil + } + r.sealed = true + now := time.Now().UTC() + ev := r.assembleLocked(now) + r.mu.Unlock() + + r.sdk.mu.Lock() + delete(r.sdk.active, r) + r.sdk.mu.Unlock() + + if r.sdk.cfg.Redactor != nil && len(ev.Fields) > 0 { + ev.Fields = r.sdk.cfg.Redactor(ev.Fields) + } + + if err := emit(r.sdk.out, ev); err != nil { + return ev, err + } + r.sdk.emitted.Add(1) + if ev.Status == eventv2.StatusSuppressed { + r.sdk.suppressed.Add(1) + } + return ev, nil +} + +func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { + ev := &eventv2.Event{ + SchemaVersion: eventv2.SchemaVersion2, + EventID: r.eventID, + TsStart: r.tsStart, + TsEnd: tsEnd, + DurationMS: int64(tsEnd.Sub(r.tsStart) / 1e6), + Kind: "http", + Service: r.sdk.cfg.Service, + Env: r.sdk.cfg.Env, + Version: r.sdk.cfg.Version, + TraceID: r.traceID, + SpanID: r.spanID, + ParentSpanID: r.parentSpanID, + Status: r.snapshotStatusLocked(), + } + + if ev.Status == eventv2.StatusSuppressed { + return ev + } + + if len(r.fields) > 0 { + // Copy only when a Redactor will mutate the result; otherwise hand + // the sealed request's map directly to the event. + if r.sdk.cfg.Redactor != nil { + ev.Fields = make(map[string]any, len(r.fields)) + maps.Copy(ev.Fields, r.fields) + } else { + ev.Fields = r.fields + r.fields = nil + } + } + + if r.anchorStep != "" && r.anchorCode != "" { + ev.Anchor = &eventv2.Anchor{Step: r.anchorStep, ErrorCode: r.anchorCode} + } + + if r.headerOnly { + // Header-heavy fallback per §4.4: identity + status + anchor + fields, + // no steps[]/logs[]/errors[]. + return ev + } + + if len(r.steps) > 0 { + ev.Steps = make([]eventv2.Step, 0, len(r.steps)) + for _, s := range r.steps { + step := eventv2.Step{ + Name: s.name, + SpanID: s.spanID, + StartMS: s.startMS, + DurationMS: s.durationMS, + Status: s.status, + } + if s.err != nil { + step.Error = s.err.toStepError() + } + ev.Steps = append(ev.Steps, step) + } + } + + if len(r.logs) > 0 { + ev.Logs = make([]eventv2.Log, 0, len(r.logs)) + for _, l := range r.logs { + ev.Logs = append(ev.Logs, eventv2.Log{ + TsOffsetMS: l.tsOffsetMS, + Level: l.level, + Msg: l.msg, + Fields: l.fields, + }) + } + } + + if len(r.errs) > 0 { + ev.Errors = make([]eventv2.ErrorRef, 0, len(r.errs)) + for _, e := range r.errs { + ev.Errors = append(ev.Errors, eventv2.ErrorRef{Code: e.code, Reason: e.reason}) + } + } + + return ev +} + +// snapshotStatusLocked computes the current status without sealing. Caller +// must hold r.mu. +func (r *request) snapshotStatusLocked() eventv2.Status { + if r.suppressed { + return eventv2.StatusSuppressed + } + if r.anchorStep != "" { + return eventv2.StatusError + } + return eventv2.StatusOK +} + +func emit(w io.Writer, ev *eventv2.Event) error { + raw, err := json.Marshal(ev) + if err != nil { + return fmt.Errorf("waylog: marshal event: %w", err) + } + raw = append(raw, '\n') + if _, err := w.Write(raw); err != nil { + return fmt.Errorf("waylog: write event: %w", err) + } + return nil +} diff --git a/pkg/waylog/v2/errors.go b/pkg/waylog/v2/errors.go new file mode 100644 index 0000000..b56a92e --- /dev/null +++ b/pkg/waylog/v2/errors.go @@ -0,0 +1,72 @@ +package waylogv2 + +import ( + "fmt" + "os" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +// Error is the structured failure value carried through Fail. +type Error struct { + Code string + Reason string + Cause string +} + +// Opt configures a NewError. +type Opt func(*Error) + +func WithReason(reason string) Opt { return func(e *Error) { e.Reason = reason } } +func WithCause(cause string) Opt { return func(e *Error) { e.Cause = cause } } + +// NewError builds a structured Error. Reserved WAYLOG_* codes are rejected +// (returns nil) and counted in StatsSnapshot.ReservedCodeRejections so misuse +// is observable. Lifecycle codes must come from SDK synthesis. +func NewError(code string, opts ...Opt) *Error { + if isReserved(code) { + recordReservedRejection(code, "NewError") + return nil + } + e := &Error{Code: code} + for _, opt := range opts { + opt(e) + } + return e +} + +func (e *Error) Error() string { + if e == nil { + return "" + } + if e.Reason != "" { + return fmt.Sprintf("%s: %s", e.Code, e.Reason) + } + return e.Code +} + +func isReserved(code string) bool { + switch code { + case eventv2.CodeTimeout, eventv2.CodeAborted, eventv2.CodePanic, eventv2.CodePartial: + return true + } + return false +} + +func recordReservedRejection(code, where string) { + s := getState() + if s == nil { + return + } + s.reservedRejected.Add(1) + if s.cfg.DevMode { + fmt.Fprintf(os.Stderr, "waylog: %s rejected reserved error code %q\n", where, code) + } +} + +func (e *Error) toStepError() *eventv2.StepError { + if e == nil { + return nil + } + return &eventv2.StepError{Code: e.Code, Reason: e.Reason, Cause: e.Cause} +} diff --git a/pkg/waylog/v2/explain.go b/pkg/waylog/v2/explain.go new file mode 100644 index 0000000..361b3c2 --- /dev/null +++ b/pkg/waylog/v2/explain.go @@ -0,0 +1,133 @@ +package waylogv2 + +import ( + "context" + "errors" + "fmt" + "strings" +) + +// ExplainResult is the canonical narrative shape (§4.1). String() renders the +// plain-text form used by the CLI and dev stderr; the same structure is +// serialized to JSON by GET /v1/traces/story (slice 3+). +type ExplainResult struct { + TraceID string + Service string + Route string + Status string + Anchor AnchorInfo + Path []StepInfo + Logs []LogInfo + Downstream []DownstreamEdge +} + +type AnchorInfo struct { + Step string + ErrorCode string +} + +type StepInfo struct { + Name string + StartMS int64 + DurationMS int64 + Status string + ErrorCode string + ErrorMsg string +} + +type LogInfo struct { + TsOffsetMS int64 + Level string + Msg string + Step string +} + +type DownstreamEdge struct { + Step string + Service string + Endpoint string +} + +// ErrNoActiveRequest is returned when Explain is called outside a request. +var ErrNoActiveRequest = errors.New("waylog: no active request") + +// Explain returns a snapshot view of the in-flight request buffer. It does +// not seal or finalize the request. +func Explain(ctx context.Context) (*ExplainResult, error) { + r := requestFromContext(ctx) + if r == nil { + return nil, ErrNoActiveRequest + } + + r.mu.Lock() + defer r.mu.Unlock() + + out := &ExplainResult{ + TraceID: r.traceID, + Service: r.sdk.cfg.Service, + Status: string(r.snapshotStatusLocked()), + } + if route, ok := routeFromFields(r.fields); ok { + out.Route = route + } + if r.anchorStep != "" || r.anchorCode != "" { + out.Anchor = AnchorInfo{Step: r.anchorStep, ErrorCode: r.anchorCode} + } + for _, s := range r.steps { + si := StepInfo{ + Name: s.name, + StartMS: s.startMS, + DurationMS: s.durationMS, + Status: s.status, + } + if s.err != nil { + si.ErrorCode = s.err.Code + si.ErrorMsg = s.err.Reason + } + out.Path = append(out.Path, si) + } + for _, l := range r.logs { + out.Logs = append(out.Logs, LogInfo{ + TsOffsetMS: l.tsOffsetMS, + Level: l.level, + Msg: l.msg, + Step: l.stepName, + }) + } + return out, nil +} + +func (r *ExplainResult) String() string { + var b strings.Builder + fmt.Fprintf(&b, "trace=%s service=%s status=%s\n", r.TraceID, r.Service, r.Status) + if r.Route != "" { + fmt.Fprintf(&b, "route=%s\n", r.Route) + } + if r.Anchor.Step != "" || r.Anchor.ErrorCode != "" { + fmt.Fprintf(&b, "anchor=%s code=%s\n", r.Anchor.Step, r.Anchor.ErrorCode) + } + for _, s := range r.Path { + if s.Status == "error" { + fmt.Fprintf(&b, " ✗ %s (%dms) %s: %s\n", s.Name, s.DurationMS, s.ErrorCode, s.ErrorMsg) + continue + } + fmt.Fprintf(&b, " · %s (%dms)\n", s.Name, s.DurationMS) + } + for _, l := range r.Logs { + if l.Step != "" { + fmt.Fprintf(&b, "[%s] [%dms] [%s] %s\n", l.Level, l.TsOffsetMS, l.Step, l.Msg) + continue + } + fmt.Fprintf(&b, "[%s] [%dms] %s\n", l.Level, l.TsOffsetMS, l.Msg) + } + return b.String() +} + +func routeFromFields(f F) (string, bool) { + httpMap, ok := f["http"].(map[string]any) + if !ok { + return "", false + } + r, _ := httpMap["route"].(string) + return r, r != "" +} diff --git a/pkg/waylog/v2/logger.go b/pkg/waylog/v2/logger.go new file mode 100644 index 0000000..dd1a03e --- /dev/null +++ b/pkg/waylog/v2/logger.go @@ -0,0 +1,106 @@ +package waylogv2 + +import ( + "context" + "maps" + "time" +) + +// Logger is the request-scoped logging surface returned by From. +type Logger interface { + Info(msg string, fields ...F) + Warn(msg string, fields ...F) + Error(msg string, err *Error, fields ...F) +} + +// From returns a request-scoped Logger. If ctx has no active Waylog request, +// the returned Logger is a no-op. +func From(ctx context.Context) Logger { + r := requestFromContext(ctx) + if r == nil { + return noopLogger{} + } + return &reqLogger{r: r} +} + +type reqLogger struct{ r *request } + +func (l *reqLogger) Info(msg string, fields ...F) { l.r.appendLog("info", msg, nil, fields) } +func (l *reqLogger) Warn(msg string, fields ...F) { l.r.appendLog("warn", msg, nil, fields) } +func (l *reqLogger) Error(msg string, err *Error, fields ...F) { + l.r.appendLog("error", msg, err, fields) +} + +type noopLogger struct{} + +func (noopLogger) Info(string, ...F) {} +func (noopLogger) Warn(string, ...F) {} +func (noopLogger) Error(string, *Error, ...F) {} + +func (r *request) appendLog(level, msg string, err *Error, more []F) { + if r == nil { + return + } + + merged := mergeFields(more) + if err != nil { + if merged == nil { + merged = F{} + } + merged["error.code"] = err.Code + if err.Reason != "" { + merged["error.reason"] = err.Reason + } + if err.Cause != "" { + merged["error.cause"] = err.Cause + } + } + + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed || r.sealed { + return + } + + stepName := "" + if n := len(r.stepStack); n > 0 { + stepName = r.stepStack[n-1] + } + + if err != nil { + r.recordErrorLocked(err.Code, err.Reason) + } + + r.addLogLocked(logBuf{ + tsOffsetMS: int64(time.Since(r.tsStart) / 1e6), + level: level, + msg: msg, + fields: merged, + stepName: stepName, + }) +} + +// mergeFields returns a freshly-allocated shallow copy of the merged maps. +// Mutating the *outer* input F after the call is safe; **nested** map/slice +// values inside the F remain caller-owned and MAY mutate the buffered log +// entry if the caller mutates them afterward. +// +// This is an intentional split from SetField (which deep-clones): logger +// calls are on the per-request hot path where deep clone cost compounds. +// Snapshot-at-call lives in SetField; the logger contract is "don't mutate +// nested objects after passing them to Info/Warn/Error". +// +// Empty inputs return nil to avoid emitting `"fields":{}`. +func mergeFields(in []F) F { + if len(in) == 0 { + return nil + } + out := F{} + for _, m := range in { + maps.Copy(out, m) + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go new file mode 100644 index 0000000..b52e850 --- /dev/null +++ b/pkg/waylog/v2/request.go @@ -0,0 +1,409 @@ +package waylogv2 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "sync" + "time" +) + +const ( + stepStatusOK = "ok" + stepStatusError = "error" +) + +type ctxKey struct{} + +// BeginOptions seeds a new request buffer with identity captured by the +// caller (HTTP middleware, test code, future framework adapters). +type BeginOptions struct { + TraceID string + SpanID string + ParentSpanID string + Now time.Time +} + +// Begin opens a request buffer and returns a context that carries it. The +// returned ctx must be passed to Finalize to assemble and emit the final +// wide event. +// +// Begin is intended for middleware and adapter authors. Application code +// should use a framework adapter once those exist. +func Begin(ctx context.Context, opts BeginOptions) context.Context { + s := getState() + if s == nil { + return ctx + } + + now := opts.Now + if now.IsZero() { + now = time.Now().UTC() + } + + traceID := opts.TraceID + if traceID == "" { + traceID = newTraceID() + } + spanID := opts.SpanID + if spanID == "" { + spanID = newSpanID() + } + + r := &request{ + sdk: s, + eventID: newEventID(), + tsStart: now, + traceID: traceID, + spanID: spanID, + parentSpanID: opts.ParentSpanID, + fields: F{}, + maxSteps: s.cfg.MaxSteps, + maxLogs: s.cfg.MaxLogs, + maxBytes: s.cfg.MaxBufferBytes, + } + + s.mu.Lock() + s.active[r] = struct{}{} + s.mu.Unlock() + + return context.WithValue(ctx, ctxKey{}, r) +} + +func requestFromContext(ctx context.Context) *request { + if ctx == nil { + return nil + } + v, _ := ctx.Value(ctxKey{}).(*request) + return v +} + +// SetField sets a top-level request field (e.g. fields.http or fields.user). +// The value is deep-cloned for `map[string]any` and `[]any` shapes, so caller +// mutation at any level after the call cannot change the emitted event. +// Logger field bags (`From(ctx).Info(msg, F{...})`) use a shallower contract. +func SetField(ctx context.Context, key string, value any) { + r := requestFromContext(ctx) + if r == nil || key == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed || r.sealed { + return + } + r.fields[key] = cloneDeep(value) +} + +func cloneDeep(v any) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + out[k] = cloneDeep(val) + } + return out + case []any: + out := make([]any, len(x)) + for i, val := range x { + out[i] = cloneDeep(val) + } + return out + } + return v +} + +type request struct { + sdk *sdk + + eventID string + tsStart time.Time + traceID string + spanID string + parentSpanID string + + maxSteps int + maxLogs int + maxBytes int + + mu sync.Mutex + + stepStack []string + steps []stepBuf + logs []logBuf + fields F + errs []errEntry + + bufBytes int + + suppressed bool + sealed bool + headerOnly bool // degraded by MaxBufferBytes pressure (§4.4) + + // First observable failing step, or "request" sentinel for a Fail() with + // no active step. Set once and never overwritten. + anchorStep string + anchorCode string +} + +type stepBuf struct { + name string + spanID string + startMS int64 + durationMS int64 + status string + err *Error +} + +type logBuf struct { + tsOffsetMS int64 + level string + msg string + fields F + stepName string // captured at log-time; used by Explain only +} + +type errEntry struct { + code string + reason string +} + +func (r *request) addStepLocked(s stepBuf) { + // Anchor and errors[] are tiny and survive buffer pressure — record + // before any drop / degrade decision. + if s.status == stepStatusError && s.err != nil { + r.recordErrorLocked(s.err.Code, s.err.Reason) + if r.anchorStep == "" { + r.anchorStep = s.name + r.anchorCode = s.err.Code + } + } + + if r.headerOnly { + r.sdk.stepsDropped.Add(1) + return + } + + bytes := stepBytes(s) + if r.bufBytes+bytes > r.maxBytes { + r.dropOkLogsLocked() + if r.bufBytes+bytes > r.maxBytes { + r.dropOkStepsLocked() + } + if r.bufBytes+bytes > r.maxBytes { + if s.status == stepStatusOK { + r.sdk.stepsDropped.Add(1) + r.sdk.bytesDropped.Add(int64(bytes)) + return + } + // Error step still won't fit → header-only fallback. + r.degradeToHeaderOnlyLocked() + r.sdk.stepsDropped.Add(1) + return + } + } + + if len(r.steps) >= r.maxSteps { + if s.status == stepStatusOK { + r.sdk.stepsDropped.Add(1) + return + } + if !r.evictOldestOkStepLocked() { + r.sdk.stepsDropped.Add(1) + return + } + } + + r.steps = append(r.steps, s) + r.bufBytes += bytes +} + +func (r *request) addLogLocked(l logBuf) { + if r.headerOnly { + r.sdk.logsDropped.Add(1) + return + } + + bytes := logBytes(l) + if r.bufBytes+bytes > r.maxBytes { + r.dropOkLogsLocked() + if r.bufBytes+bytes > r.maxBytes { + if l.level == "info" { + r.sdk.logsDropped.Add(1) + r.sdk.bytesDropped.Add(int64(bytes)) + return + } + // warn/error log still won't fit → header-only fallback. + r.degradeToHeaderOnlyLocked() + r.sdk.logsDropped.Add(1) + return + } + } + + if len(r.logs) >= r.maxLogs { + if l.level == "info" { + r.sdk.logsDropped.Add(1) + return + } + if !r.evictOldestInfoLogLocked() { + r.sdk.logsDropped.Add(1) + return + } + } + + r.logs = append(r.logs, l) + r.bufBytes += bytes +} + +// degradeToHeaderOnlyLocked discards buffered detail and switches the request +// to header-only emission. Anchor + fields + errs (already recorded) survive. +func (r *request) degradeToHeaderOnlyLocked() { + if r.headerOnly { + return + } + r.headerOnly = true + r.sdk.bufferOverflows.Add(1) + for _, l := range r.logs { + r.sdk.bytesDropped.Add(int64(logBytes(l))) + } + for _, s := range r.steps { + r.sdk.bytesDropped.Add(int64(stepBytes(s))) + } + r.steps = nil + r.logs = nil + r.bufBytes = 0 +} + +func (r *request) recordErrorLocked(code, reason string) { + if code == "" { + return + } + for _, e := range r.errs { + if e.code == code { + return + } + } + r.errs = append(r.errs, errEntry{code: code, reason: reason}) +} + +func (r *request) dropOkLogsLocked() { + if len(r.logs) == 0 { + return + } + kept := r.logs[:0] + for _, l := range r.logs { + if l.level == "info" { + b := logBytes(l) + r.bufBytes -= b + r.sdk.logsDropped.Add(1) + r.sdk.bytesDropped.Add(int64(b)) + continue + } + kept = append(kept, l) + } + r.logs = kept +} + +func (r *request) dropOkStepsLocked() { + if len(r.steps) == 0 { + return + } + kept := r.steps[:0] + for _, s := range r.steps { + if s.status == stepStatusOK && s.name != r.anchorStep { + b := stepBytes(s) + r.bufBytes -= b + r.sdk.stepsDropped.Add(1) + r.sdk.bytesDropped.Add(int64(b)) + continue + } + kept = append(kept, s) + } + r.steps = kept +} + +func (r *request) evictOldestOkStepLocked() bool { + for i, s := range r.steps { + if s.status == stepStatusOK && s.name != r.anchorStep { + r.bufBytes -= stepBytes(s) + r.steps = append(r.steps[:i], r.steps[i+1:]...) + r.sdk.stepsDropped.Add(1) + return true + } + } + return false +} + +func (r *request) evictOldestInfoLogLocked() bool { + for i, l := range r.logs { + if l.level == "info" { + r.bufBytes -= logBytes(l) + r.logs = append(r.logs[:i], r.logs[i+1:]...) + r.sdk.logsDropped.Add(1) + return true + } + } + return false +} + +func newEventID() string { return newUUIDv4() } +func newTraceID() string { return randomHex(16) } +func newSpanID() string { return randomHex(8) } + +// newUUIDv4 formats a v4 UUID without intermediate string allocations. +func newUUIDv4() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + seedFromTime(b[:]) + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + + var dst [36]byte + hex.Encode(dst[0:8], b[0:4]) + dst[8] = '-' + hex.Encode(dst[9:13], b[4:6]) + dst[13] = '-' + hex.Encode(dst[14:18], b[6:8]) + dst[18] = '-' + hex.Encode(dst[19:23], b[8:10]) + dst[23] = '-' + hex.Encode(dst[24:36], b[10:16]) + return string(dst[:]) +} + +func randomHex(n int) string { + var buf [16]byte + if _, err := rand.Read(buf[:n]); err != nil { + seedFromTime(buf[:n]) + } + return hex.EncodeToString(buf[:n]) +} + +func seedFromTime(b []byte) { + nano := time.Now().UnixNano() + for i := range b { + b[i] = byte(nano >> (i % 8 * 8)) + } +} + +// Conservative byte-size estimates for cap accounting; not exact JSON size. +// A stable upper bound is enough for the buffer-pressure cascade. +func stepBytes(s stepBuf) int { + n := 32 + len(s.name) + len(s.spanID) + if s.err != nil { + n += 16 + len(s.err.Code) + len(s.err.Reason) + len(s.err.Cause) + } + return n +} + +func logBytes(l logBuf) int { + n := 24 + len(l.level) + len(l.msg) + for k, v := range l.fields { + n += len(k) + 8 + if str, ok := v.(string); ok { + n += len(str) + } + } + return n +} diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go new file mode 100644 index 0000000..c6e621d --- /dev/null +++ b/pkg/waylog/v2/sdk_test.go @@ -0,0 +1,651 @@ +package waylogv2 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +const schemaPath = "../../../docs/schema/v2.0.json" + +type harness struct { + t *testing.T + buf *bytes.Buffer +} + +func newHarness(t *testing.T, cfg Config) *harness { + t.Helper() + resetForTest() + buf := &bytes.Buffer{} + if cfg.Service == "" { + cfg.Service = "checkout" + } + if cfg.Env == "" { + cfg.Env = "test" + } + cfg.Output = buf + if err := Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return &harness{t: t, buf: buf} +} + +func (h *harness) lastEvent() *eventv2.Event { + h.t.Helper() + out := h.buf.Bytes() + if len(out) == 0 { + return nil + } + lines := bytes.Split(bytes.TrimRight(out, "\n"), []byte{'\n'}) + last := lines[len(lines)-1] + var ev eventv2.Event + if err := json.Unmarshal(last, &ev); err != nil { + h.t.Fatalf("unmarshal emitted event: %v\nraw=%s", err, last) + } + return &ev +} + +func (h *harness) validateLast() { + h.t.Helper() + ev := h.lastEvent() + if ev == nil { + h.t.Fatal("no event emitted") + } + if err := eventv2.Validate(schemaPath, ev); err != nil { + raw, _ := json.MarshalIndent(ev, "", " ") + h.t.Fatalf("emitted event fails v2.0 schema: %v\n%s", err, raw) + } +} + +func TestInitRequiresServiceAndEnv(t *testing.T) { + resetForTest() + if err := Init(Config{}); err == nil { + t.Fatal("expected error when Service is empty") + } + resetForTest() + if err := Init(Config{Service: "x"}); err == nil { + t.Fatal("expected error when Env is empty") + } +} + +func TestInitRefusesReinitWithActiveRequests(t *testing.T) { + newHarness(t, Config{}) + _ = Begin(context.Background(), BeginOptions{}) + // Don't reset; intentionally leave one request active. + err := Init(Config{Service: "x", Env: "test"}) + if !errors.Is(err, ErrAlreadyInitialized) { + t.Fatalf("expected ErrAlreadyInitialized, got %v", err) + } +} + +func TestInitAllowedAfterDrain(t *testing.T) { + newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + if _, err := Finalize(ctx); err != nil { + t.Fatal(err) + } + // Active count back to zero — re-init must succeed. + if err := Init(Config{Service: "y", Env: "test", Output: &bytes.Buffer{}}); err != nil { + t.Fatalf("re-init after drain: %v", err) + } +} + +func TestOKRequestEmitsValidEvent(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + SetField(ctx, "http", map[string]any{"method": "GET", "route": "/health", "status": 200}) + From(ctx).Info("served", F{"latency_ms": 4}) + + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusOK { + t.Fatalf("status=%s want ok", ev.Status) + } + if ev.Anchor != nil { + t.Fatalf("ok event must not carry anchor: %+v", ev.Anchor) + } + if len(ev.Logs) != 1 || ev.Logs[0].Msg != "served" { + t.Fatalf("log not buffered: %+v", ev.Logs) + } + if ev.Fields["http"] == nil { + t.Fatalf("fields.http missing: %+v", ev.Fields) + } + if got := Stats().EventsEmitted; got != 1 { + t.Fatalf("EventsEmitted=%d want 1", got) + } +} + +func TestStepFailureSynthesizesAnchor(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + werr := NewError("PMT_502", WithReason("upstream gateway 5xx")) + _ = StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + return werr + }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "payment.charge" || ev.Anchor.ErrorCode != "PMT_502" { + t.Fatalf("anchor wrong: %+v", ev.Anchor) + } + if len(ev.Steps) != 1 || ev.Steps[0].Status != "error" || ev.Steps[0].Error == nil || + ev.Steps[0].Error.Code != "PMT_502" { + t.Fatalf("step failure not recorded: %+v", ev.Steps) + } + if len(ev.Errors) != 1 || ev.Errors[0].Code != "PMT_502" { + t.Fatalf("errors[] not deduped/recorded: %+v", ev.Errors) + } +} + +func TestExplicitFailWithNoActiveStepUsesRequestSentinel(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + Fail(ctx, NewError("AUTH_DENIED")) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != "AUTH_DENIED" { + t.Fatalf("anchor wrong on bare Fail: %+v", ev.Anchor) + } +} + +func TestFirstFailingStepWinsAnchor(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + _ = StepVoid(ctx, "first", func(ctx context.Context) error { + return NewError("FIRST_ERR") + }) + _ = StepVoid(ctx, "second", func(ctx context.Context) error { + return NewError("SECOND_ERR") + }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Anchor.Step != "first" || ev.Anchor.ErrorCode != "FIRST_ERR" { + t.Fatalf("first failure must own anchor: %+v", ev.Anchor) + } + if len(ev.Errors) != 2 { + t.Fatalf("errors[] should contain both codes deduped: %+v", ev.Errors) + } +} + +func TestSuppressEmitsHeaderOnlyEvent(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + From(ctx).Info("first log") + Suppress(ctx) + From(ctx).Warn("after suppress") + SetField(ctx, "user", map[string]any{"id": "u_1"}) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusSuppressed { + t.Fatalf("status=%s want suppressed", ev.Status) + } + if len(ev.Steps) != 0 || len(ev.Logs) != 0 { + t.Fatalf("suppressed event must have empty steps/logs: %+v / %+v", ev.Steps, ev.Logs) + } + if ev.Anchor != nil { + t.Fatalf("suppressed event must not carry anchor: %+v", ev.Anchor) + } + stats := Stats() + if stats.EventsEmitted != 1 || stats.EventsSuppressed != 1 { + t.Fatalf("suppressed must increment both EventsEmitted and EventsSuppressed: %+v", stats) + } +} + +func TestSuppressThenFailKeepsSuppressedAndCounts(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + Suppress(ctx) + Fail(ctx, NewError("LATE_FAIL")) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusSuppressed { + t.Fatalf("status=%s want suppressed", ev.Status) + } + if got := Stats().SuppressedThenFailed; got != 1 { + t.Fatalf("SuppressedThenFailed=%d want 1", got) + } +} + +func TestNewErrorRejectsReservedCodes(t *testing.T) { + newHarness(t, Config{}) + for _, code := range []string{eventv2.CodeTimeout, eventv2.CodeAborted, eventv2.CodePanic, eventv2.CodePartial} { + got := NewError(code) + if got != nil { + t.Fatalf("NewError(%q) must return nil; got %+v", code, got) + } + } + if got := Stats().ReservedCodeRejections; got != 4 { + t.Fatalf("ReservedCodeRejections=%d want 4", got) + } +} + +func TestFailRejectsHandCraftedReservedCode(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + // Bypass NewError by hand-crafting *Error. + Fail(ctx, &Error{Code: eventv2.CodeTimeout, Reason: "tried to spoof"}) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusOK { + t.Fatalf("hand-crafted reserved code must not become anchor; status=%s", ev.Status) + } + if got := Stats().ReservedCodeRejections; got != 1 { + t.Fatalf("ReservedCodeRejections=%d want 1", got) + } +} + +func TestStepReturningReservedCodeIsRewrittenToERR(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + _ = StepVoid(ctx, "shady", func(ctx context.Context) error { + return &Error{Code: eventv2.CodePanic, Reason: "user spoof"} + }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Anchor == nil || ev.Anchor.ErrorCode != "ERR" { + t.Fatalf("reserved code from user step must collapse to ERR: %+v", ev.Anchor) + } + if got := Stats().ReservedCodeRejections; got != 1 { + t.Fatalf("ReservedCodeRejections=%d want 1 (Step path must count rejection)", got) + } +} + +func TestNoActiveRequestNoOps(t *testing.T) { + resetForTest() + if err := Init(Config{Service: "x", Env: "test"}); err != nil { + t.Fatal(err) + } + ctx := context.Background() + + From(ctx).Info("nope") + From(ctx).Error("nope", NewError("X")) + Suppress(ctx) + Fail(ctx, NewError("Y")) + if _, err := Explain(ctx); !errors.Is(err, ErrNoActiveRequest) { + t.Fatalf("Explain on bare ctx must return ErrNoActiveRequest, got %v", err) + } +} + +func TestStepPropagatesNonWaylogError(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + plain := errors.New("disk full") + _ = StepVoid(ctx, "db.write", func(ctx context.Context) error { return plain }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Anchor == nil || ev.Anchor.Step != "db.write" || ev.Anchor.ErrorCode != "ERR" { + t.Fatalf("plain error must produce ERR-coded anchor: %+v", ev.Anchor) + } + if ev.Steps[0].Error.Reason != "disk full" { + t.Fatalf("plain error reason not preserved: %+v", ev.Steps[0].Error) + } +} + +func TestMaxLogsDropsInfoFirst(t *testing.T) { + h := newHarness(t, Config{MaxLogs: 4}) + ctx := Begin(context.Background(), BeginOptions{}) + + From(ctx).Info("i1") + From(ctx).Info("i2") + From(ctx).Warn("w1") + From(ctx).Info("i3") + From(ctx).Error("e1", NewError("X")) + From(ctx).Info("i4") + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if len(ev.Logs) != 4 { + t.Fatalf("want 4 logs after cap, got %d: %+v", len(ev.Logs), ev.Logs) + } + for _, l := range ev.Logs { + if l.Msg == "i4" { + t.Fatalf("dropped info should not be present: %+v", ev.Logs) + } + } + hasWarn, hasError := false, false + for _, l := range ev.Logs { + if l.Level == "warn" { + hasWarn = true + } + if l.Level == "error" { + hasError = true + } + } + if !hasWarn || !hasError { + t.Fatalf("warn/error logs must be preserved under cap: %+v", ev.Logs) + } + if got := Stats().LogsDropped; got < 1 { + t.Fatalf("LogsDropped counter not incremented: %d", got) + } +} + +func TestMaxStepsDropsOkFirst(t *testing.T) { + h := newHarness(t, Config{MaxSteps: 3}) + ctx := Begin(context.Background(), BeginOptions{}) + + _ = StepVoid(ctx, "ok1", func(ctx context.Context) error { return nil }) + _ = StepVoid(ctx, "ok2", func(ctx context.Context) error { return nil }) + _ = StepVoid(ctx, "ok3", func(ctx context.Context) error { return nil }) + _ = StepVoid(ctx, "boom", func(ctx context.Context) error { return NewError("BOOM") }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if len(ev.Steps) != 3 { + t.Fatalf("want 3 steps after cap, got %d: %+v", len(ev.Steps), ev.Steps) + } + hasBoom := false + for _, s := range ev.Steps { + if s.Name == "boom" { + hasBoom = true + } + } + if !hasBoom { + t.Fatalf("error step must be present after eviction: %+v", ev.Steps) + } +} + +func TestMaxStepsDropsIncomingOkWhenFull(t *testing.T) { + h := newHarness(t, Config{MaxSteps: 2}) + ctx := Begin(context.Background(), BeginOptions{}) + + _ = StepVoid(ctx, "boom", func(ctx context.Context) error { return NewError("BOOM") }) + _ = StepVoid(ctx, "ok1", func(ctx context.Context) error { return nil }) + _ = StepVoid(ctx, "ok2", func(ctx context.Context) error { return nil }) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if len(ev.Steps) != 2 { + t.Fatalf("want 2 steps, got %d: %+v", len(ev.Steps), ev.Steps) + } + for _, s := range ev.Steps { + if s.Name == "ok2" { + t.Fatalf("incoming ok step should be dropped, not buffered: %+v", ev.Steps) + } + } + if got := Stats().StepsDropped; got < 1 { + t.Fatalf("StepsDropped not incremented: %d", got) + } +} + +func TestMaxBufferBytesDegradesToHeaderOnly(t *testing.T) { + h := newHarness(t, Config{MaxBufferBytes: 256, MaxLogs: 1024, MaxSteps: 1024}) + ctx := Begin(context.Background(), BeginOptions{}) + + bigPayload := strings.Repeat("x", 512) + From(ctx).Warn("big-warn-1", F{"blob": bigPayload}) + From(ctx).Warn("big-warn-2", F{"blob": bigPayload}) + + Fail(ctx, NewError("BUDGET_BLOWN")) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if len(ev.Logs) != 0 || len(ev.Steps) != 0 { + t.Fatalf("header-heavy fallback must drop steps/logs: steps=%d logs=%d", len(ev.Steps), len(ev.Logs)) + } + if ev.Anchor == nil || ev.Anchor.ErrorCode != "BUDGET_BLOWN" { + t.Fatalf("anchor must survive header-only fallback: %+v", ev.Anchor) + } + if got := Stats().BufferOverflows; got != 1 { + t.Fatalf("BufferOverflows=%d want 1", got) + } +} + +func TestRedactorMutatesEventFields(t *testing.T) { + h := newHarness(t, Config{ + Redactor: func(f F) F { + if u, ok := f["user"].(map[string]any); ok { + u["id"] = "REDACTED" + f["user"] = u + } + return f + }, + }) + ctx := Begin(context.Background(), BeginOptions{}) + SetField(ctx, "user", map[string]any{"id": "u_secret"}) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + user, _ := ev.Fields["user"].(map[string]any) + if user["id"] != "REDACTED" { + t.Fatalf("redactor not applied: %+v", ev.Fields) + } +} + +func TestLoggerFieldsImmuneToCallerMutation(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + fields := F{"latency_ms": 4, "route": "/x"} + From(ctx).Info("served", fields) + fields["latency_ms"] = 99999 + + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if len(ev.Logs) != 1 { + t.Fatalf("want 1 log, got %d", len(ev.Logs)) + } + got := ev.Logs[0].Fields + // JSON unmarshal returns float64 for numbers. + if v, _ := got["latency_ms"].(float64); v != 4 { + t.Fatalf("caller mutation leaked into log fields: latency_ms=%v want 4", got["latency_ms"]) + } +} + +func TestSetFieldDeepClonesNestedMapsAndSlices(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + payload := map[string]any{ + "route": "/x", + "nested": map[string]any{"deep": "v1"}, + "list": []any{map[string]any{"k": "v1"}}, + } + SetField(ctx, "http", payload) + + payload["route"] = "/y" + payload["nested"].(map[string]any)["deep"] = "MUTATED" + payload["list"].([]any)[0].(map[string]any)["k"] = "MUTATED" + + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + got, _ := ev.Fields["http"].(map[string]any) + if got["route"] != "/x" { + t.Fatalf("top-level mutation leaked: route=%v want /x", got["route"]) + } + if nested, _ := got["nested"].(map[string]any); nested["deep"] != "v1" { + t.Fatalf("nested-map mutation leaked: deep=%v want v1", nested["deep"]) + } + list, _ := got["list"].([]any) + if first, _ := list[0].(map[string]any); first["k"] != "v1" { + t.Fatalf("nested-slice element mutation leaked: k=%v want v1", first["k"]) + } +} + +func TestLoggerFieldsHonorShallowNestedContract(t *testing.T) { + // Pins the documented shallow contract for the per-call hot path: + // mergeFields copies the outer F but does NOT walk nested maps/slices. + // Snapshot-at-call lives in SetField; logger callers must not mutate + // nested objects after passing them. If this contract changes (e.g. + // promoted to deep clone), update mergeFields' doc and flip this test. + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + nested := map[string]any{"route": "/x"} + From(ctx).Info("served", F{"http": nested}) + nested["route"] = "/y" + + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + got, _ := ev.Logs[0].Fields["http"].(map[string]any) + if got["route"] != "/y" { + t.Fatalf("logger nested contract regressed to deep clone: route=%v want /y", got["route"]) + } +} + +func TestExplainSnapshotMidRequest(t *testing.T) { + newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + SetField(ctx, "http", map[string]any{"route": "/checkout"}) + From(ctx).Info("step1") + _ = StepVoid(ctx, "db.load_cart", func(ctx context.Context) error { return nil }) + + res, err := Explain(ctx) + if err != nil { + t.Fatal(err) + } + if res.Route != "/checkout" || res.Status != "ok" { + t.Fatalf("unexpected snapshot: %+v", res) + } + if len(res.Path) != 1 || res.Path[0].Name != "db.load_cart" { + t.Fatalf("path missing closed step: %+v", res.Path) + } + if !strings.Contains(res.String(), "db.load_cart") { + t.Fatalf("String() should include step name: %s", res.String()) + } + + _, _ = Finalize(ctx) +} + +func TestConcurrentLoggingDoesNotRace(t *testing.T) { + newHarness(t, Config{MaxLogs: 100}) + ctx := Begin(context.Background(), BeginOptions{}) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for j := 0; j < 20; j++ { + From(ctx).Info("hello", F{"n": n, "j": j}) + } + }(i) + } + wg.Wait() + + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } +} + +func TestFinalizeTwiceCountsLateCompletion(t *testing.T) { + newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + if _, err := Finalize(ctx); err != nil { + t.Fatal(err) + } + if _, err := Finalize(ctx); err != nil { + t.Fatalf("second Finalize must be a no-op, got %v", err) + } + if got := Stats().LateCompletionAfterEmit; got != 1 { + t.Fatalf("LateCompletionAfterEmit=%d want 1", got) + } +} + +func TestShutdownReturnsAfterAllRequestsFinalize(t *testing.T) { + newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + go func() { + time.Sleep(10 * time.Millisecond) + _, _ = Finalize(ctx) + }() + + deadline, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + if err := Shutdown(deadline); err != nil { + t.Fatalf("Shutdown: %v", err) + } + if got := Stats().ActiveRequests; got != 0 { + t.Fatalf("ActiveRequests=%d want 0", got) + } +} + +func TestEmptyStepNameRunsFnButOpensNoStep(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + called := false + _ = StepVoid(ctx, "", func(ctx context.Context) error { + called = true + return nil + }) + _, _ = Finalize(ctx) + + if !called { + t.Fatal("Step with empty name must still call fn") + } + h.validateLast() + ev := h.lastEvent() + if len(ev.Steps) != 0 { + t.Fatalf("empty-name step must not appear in steps[]: %+v", ev.Steps) + } +} + +func TestFailWithEmptyCodeIsNoOp(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + Fail(ctx, &Error{Reason: "no code"}) + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusOK { + t.Fatalf("Fail with empty code must be a no-op; got status=%s", ev.Status) + } +} + +func TestShutdownTimesOutWithStuckRequest(t *testing.T) { + newHarness(t, Config{}) + _ = Begin(context.Background(), BeginOptions{}) + deadline, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := Shutdown(deadline); err == nil { + t.Fatal("expected timeout error") + } +} diff --git a/pkg/waylog/v2/step.go b/pkg/waylog/v2/step.go new file mode 100644 index 0000000..2873082 --- /dev/null +++ b/pkg/waylog/v2/step.go @@ -0,0 +1,149 @@ +package waylogv2 + +import ( + "context" + "errors" + "time" +) + +// Step runs fn as a named span within the active request. The returned T and +// error are passed through verbatim. If fn returns an error, the step records +// it and (on first failure) becomes the request anchor. +// +// If ctx has no active Waylog request, fn runs and Step is a thin pass-through. +// If name is empty, fn runs without opening a step (the empty name would +// collide with the "no anchor yet" sentinel). +func Step[T any](ctx context.Context, name string, fn func(ctx context.Context) (T, error)) (T, error) { + r := requestFromContext(ctx) + if r == nil || name == "" { + return fn(ctx) + } + r.pushStep(name) + + startedAt := time.Now() + startMS := int64(startedAt.Sub(r.tsStart) / 1e6) + v, err := fn(ctx) + dur := int64(time.Since(startedAt) / 1e6) + + r.closeStep(name, startMS, dur, err) + return v, err +} + +// StepVoid is the no-return-value form of Step. +func StepVoid(ctx context.Context, name string, fn func(ctx context.Context) error) error { + _, err := Step[struct{}](ctx, name, func(ctx context.Context) (struct{}, error) { + return struct{}{}, fn(ctx) + }) + return err +} + +// Fail explicitly marks the request as failed with the given Error. It does +// not abort the handler. Multiple Fail calls within one request are allowed; +// the first one wins for anchor purposes. +// +// Reserved WAYLOG_* codes are rejected (counted in +// StatsSnapshot.ReservedCodeRejections); the call is otherwise a no-op. +// +// On a suppressed request, Fail increments the suppressed_then_failed counter +// for visibility but does not change the emitted event. +func Fail(ctx context.Context, err *Error) { + r := requestFromContext(ctx) + if r == nil || err == nil || err.Code == "" { + return + } + if isReserved(err.Code) { + recordReservedRejection(err.Code, "Fail") + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed { + r.sdk.suppressFailed.Add(1) + return + } + if r.sealed { + return + } + r.recordErrorLocked(err.Code, err.Reason) + if r.anchorStep == "" { + if n := len(r.stepStack); n > 0 { + r.anchorStep = r.stepStack[n-1] + } else { + r.anchorStep = "request" + } + r.anchorCode = err.Code + } +} + +// Suppress marks the request as excluded from detailed buffering. Suppress is +// idempotent and non-reversible within a single request. Subsequent Fail +// calls only increment the suppressed_then_failed counter. +func Suppress(ctx context.Context) { + r := requestFromContext(ctx) + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed { + return + } + r.suppressed = true + r.steps = nil + r.logs = nil + r.errs = nil + r.bufBytes = 0 + r.headerOnly = false + r.anchorStep = "" + r.anchorCode = "" +} + +func (r *request) pushStep(name string) { + r.mu.Lock() + r.stepStack = append(r.stepStack, name) + r.mu.Unlock() +} + +func (r *request) closeStep(name string, startMS, durationMS int64, fnErr error) { + status := stepStatusOK + var werr *Error + if fnErr != nil { + status = stepStatusError + werr = &Error{Code: "ERR", Reason: fnErr.Error()} + var asWaylog *Error + if errors.As(fnErr, &asWaylog) { + if isReserved(asWaylog.Code) { + recordReservedRejection(asWaylog.Code, "Step") + } else { + werr = asWaylog + } + } + } + + r.mu.Lock() + defer r.mu.Unlock() + r.popStackLocked(name) + if r.suppressed || r.sealed { + return + } + r.addStepLocked(stepBuf{ + name: name, + startMS: startMS, + durationMS: durationMS, + status: status, + err: werr, + }) +} + +func (r *request) popStackLocked(name string) { + if n := len(r.stepStack); n > 0 && r.stepStack[n-1] == name { + r.stepStack = r.stepStack[:n-1] + return + } + for i := len(r.stepStack) - 1; i >= 0; i-- { + if r.stepStack[i] == name { + r.stepStack = append(r.stepStack[:i], r.stepStack[i+1:]...) + return + } + } +} diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go new file mode 100644 index 0000000..157460e --- /dev/null +++ b/pkg/waylog/v2/waylog.go @@ -0,0 +1,203 @@ +// Package waylogv2 is the Waylog v2.0 SDK core (local mode). +// +// Public surface: Init, Shutdown, Stats, Logger, From, Step, StepVoid, Fail, +// NewError, Suppress, Explain. Internal-but-exported helpers for middleware +// and adapter authors: Begin, Finalize, SetField. +package waylogv2 + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sync" + "sync/atomic" + "time" +) + +// F is the field bag used by Logger calls and request fields. +type F = map[string]any + +// Config configures the SDK. Set once via Init. +type Config struct { + Service string + Env string + Version string + + // Output is the writer for final wide events. Defaults to os.Stderr if + // nil and IngestURL is empty. One JSON event per line. + Output io.Writer + + // IngestURL / APIKey are accepted for API stability with future slices + // but ignored in local mode. + IngestURL string + APIKey string + + DevMode bool + + MaxSteps int + MaxLogs int + MaxRequestAge time.Duration + MaxBufferBytes int + MaxInFlightBytes int64 + MaxEventsPerSec int + + Redactor func(F) F +} + +const ( + defaultMaxSteps = 128 + defaultMaxLogs = 256 + defaultMaxBufferBytes = 512 * 1024 +) + +// StatsSnapshot is a point-in-time view of SDK counters. +// +// Go disallows a type and a free function sharing a name at package scope, so +// the spec's `func Stats() Stats` is rendered here as `func Stats() StatsSnapshot`. +type StatsSnapshot struct { + ActiveRequests int64 + EventsEmitted int64 // total final events successfully written (includes suppressed) + EventsSuppressed int64 // subset of EventsEmitted with status=suppressed + StepsDropped int64 + LogsDropped int64 + BytesDroppedFromBuffer int64 + BufferOverflows int64 // requests degraded to header-only by MaxBufferBytes pressure + ReservedCodeRejections int64 // NewError/Fail/Step returns with WAYLOG_* codes + SuppressedThenFailed int64 + LateCompletionAfterEmit int64 +} + +// ErrAlreadyInitialized is returned by Init when a prior SDK is still alive +// with active requests. Call Shutdown first, or wait for requests to drain. +var ErrAlreadyInitialized = errors.New("waylog: SDK already initialized with active requests") + +type sdk struct { + cfg Config + out io.Writer + + mu sync.Mutex + active map[*request]struct{} + + emitted atomic.Int64 + suppressed atomic.Int64 + stepsDropped atomic.Int64 + logsDropped atomic.Int64 + bytesDropped atomic.Int64 + bufferOverflows atomic.Int64 + reservedRejected atomic.Int64 + suppressFailed atomic.Int64 + lateAfterEmit atomic.Int64 +} + +var ( + stateMu sync.RWMutex + state *sdk +) + +// Init configures the SDK. Returns ErrAlreadyInitialized if a prior SDK is +// still alive with active requests; in that case, call Shutdown first. +func Init(cfg Config) error { + if cfg.Service == "" { + return errors.New("waylog: Config.Service is required") + } + if cfg.Env == "" { + return errors.New("waylog: Config.Env is required") + } + + if cfg.MaxSteps <= 0 { + cfg.MaxSteps = defaultMaxSteps + } + if cfg.MaxLogs <= 0 { + cfg.MaxLogs = defaultMaxLogs + } + if cfg.MaxBufferBytes <= 0 { + cfg.MaxBufferBytes = defaultMaxBufferBytes + } + + out := cfg.Output + if out == nil { + out = os.Stderr + } + + s := &sdk{ + cfg: cfg, + out: out, + active: make(map[*request]struct{}), + } + + stateMu.Lock() + defer stateMu.Unlock() + if state != nil { + state.mu.Lock() + n := len(state.active) + state.mu.Unlock() + if n > 0 { + return fmt.Errorf("%w: %d in flight", ErrAlreadyInitialized, n) + } + } + state = s + return nil +} + +// Shutdown waits up to ctx's deadline for in-flight requests to finalize. It +// does not force-finalize requests; middleware (or test code) is responsible +// for ending requests it started. +func Shutdown(ctx context.Context) error { + s := getState() + if s == nil { + return nil + } + + tick := time.NewTicker(5 * time.Millisecond) + defer tick.Stop() + for { + s.mu.Lock() + n := len(s.active) + s.mu.Unlock() + if n == 0 { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("waylog: shutdown timeout with %d active requests: %w", n, ctx.Err()) + case <-tick.C: + } + } +} + +// Stats returns a snapshot of runtime counters. +func Stats() StatsSnapshot { + s := getState() + if s == nil { + return StatsSnapshot{} + } + s.mu.Lock() + active := int64(len(s.active)) + s.mu.Unlock() + return StatsSnapshot{ + ActiveRequests: active, + EventsEmitted: s.emitted.Load(), + EventsSuppressed: s.suppressed.Load(), + StepsDropped: s.stepsDropped.Load(), + LogsDropped: s.logsDropped.Load(), + BytesDroppedFromBuffer: s.bytesDropped.Load(), + BufferOverflows: s.bufferOverflows.Load(), + ReservedCodeRejections: s.reservedRejected.Load(), + SuppressedThenFailed: s.suppressFailed.Load(), + LateCompletionAfterEmit: s.lateAfterEmit.Load(), + } +} + +func getState() *sdk { + stateMu.RLock() + defer stateMu.RUnlock() + return state +} + +func resetForTest() { + stateMu.Lock() + state = nil + stateMu.Unlock() +} From 004a8d6e4aab38fa6fb0445db75523e45c6620bc Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 01:56:02 -0400 Subject: [PATCH 2/8] feat(sdk): added v2 net/http middleware and watchdog lifecycle handling This change introduces a new net/http adapter at pkg/waylog/http (package wayloghttp) and wires the existing local-mode SDK core into real HTTP request lifecycles: - open a v2 request buffer per incoming HTTP request - preserve inbound W3C traceparent when present - fall back through x-trace-id, x-span-id, x-request-id, then fresh generated trace/span ids - seed and maintain fields.http.method, fields.http.route, and fields.http.status - finalize exactly once across normal return, panic recovery, cooperative cancellation, and watchdog timeout - recover panics in middleware, emit WAYLOG_PANIC, and return 500 only when headers were not already written - honor MaxRequestAge with a watchdog that seals first using status=timeout and WAYLOG_TIMEOUT - increment LateCompletionAfterEmit when a timeout-owned emit wins before handler completion - preserve ResponseWriter compatibility for Flusher, Hijacker, Pusher, ReaderFrom, and Unwrap The v2 core is extended so adapters can seal requests with lifecycle-specific terminal states without duplicating event assembly logic: - add FinalizePanic, FinalizeAborted, and FinalizeTimeout alongside normal Finalize - add internal lifecycle status synthesis for panic, aborted, and timeout paths - add SetHTTPStatus for middleware-owned response metadata updates - add MaxRequestAge accessor for adapter watchdog wiring - keep suppression precedence intact across panic, cancel, and timeout paths Tests added and updated: - core v2 regression coverage for panic/aborted lifecycle sealing and HTTP status field updates - middleware coverage for ok requests, route selection, write-without-header, trace propagation, trace fallback behavior, panic handling, cancellation, suppression precedence, watchdog timeout, timeout anchor selection, and ResponseWriter interface preservation This slice intentionally does not add outgoing RoundTripper trace injection, framework-specific adapters (chi/gin/echo), dev dual-emit, or ingest transport. The existing v1 pkg/http adapter remains untouched during the bridge window. --- pkg/waylog/http/middleware.go | 196 +++++++++++ pkg/waylog/http/middleware_test.go | 508 +++++++++++++++++++++++++++++ pkg/waylog/v2/assemble.go | 50 +++ pkg/waylog/v2/request.go | 50 ++- pkg/waylog/v2/sdk_test.go | 51 +++ pkg/waylog/v2/step.go | 1 + pkg/waylog/v2/waylog.go | 13 +- 7 files changed, 865 insertions(+), 4 deletions(-) create mode 100644 pkg/waylog/http/middleware.go create mode 100644 pkg/waylog/http/middleware_test.go diff --git a/pkg/waylog/http/middleware.go b/pkg/waylog/http/middleware.go new file mode 100644 index 0000000..9eb8eb4 --- /dev/null +++ b/pkg/waylog/http/middleware.go @@ -0,0 +1,196 @@ +package wayloghttp + +import ( + "bufio" + "context" + "io" + "net" + "net/http" + "sync/atomic" + "time" + + "github.com/sssmaran/WaylogCLI/pkg/trace" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const ( + headerTraceparent = "traceparent" + headerTraceID = "x-trace-id" + headerSpanID = "x-span-id" + headerRequestID = "x-request-id" +) + +// HTTP wraps a net/http handler so each request produces exactly one final +// Waylog v2.0 wide event following the middleware lifecycle rules. +func HTTP(next http.Handler) http.Handler { + if next == nil { + next = http.NotFoundHandler() + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + traceID, spanID, parentSpanID := resolveTraceContext(r) + ctx := waylogv2.Begin(r.Context(), waylogv2.BeginOptions{ + TraceID: traceID, + SpanID: spanID, + ParentSpanID: parentSpanID, + }) + + route := r.Pattern + if route == "" { + route = r.URL.Path + } + waylogv2.SetField(ctx, "http", waylogv2.F{ + "method": r.Method, + "route": route, + "status": http.StatusOK, + }) + + sw := wrapResponseWriter(w, ctx) + var sealed atomic.Bool + + deliver := func(kind lifecycleKind) { + if !sealed.CompareAndSwap(false, true) { + _, _ = waylogv2.Finalize(ctx) + return + } + switch kind { + case lifecycleTimeout: + _, _ = waylogv2.FinalizeTimeout(ctx) + case lifecyclePanic: + _, _ = waylogv2.FinalizePanic(ctx) + case lifecycleAborted: + _, _ = waylogv2.FinalizeAborted(ctx) + default: + _, _ = waylogv2.Finalize(ctx) + } + } + + if d := waylogv2.MaxRequestAge(); d > 0 { + timer := time.AfterFunc(d, func() { + deliver(lifecycleTimeout) + }) + defer timer.Stop() + } + + defer func() { + if rec := recover(); rec != nil { + if !sw.WroteHeader() { + sw.WriteHeader(http.StatusInternalServerError) + } + deliver(lifecyclePanic) + } + }() + + next.ServeHTTP(sw, r.WithContext(ctx)) + + if ctx.Err() != nil { + deliver(lifecycleAborted) + return + } + + deliver(lifecycleNormal) + }) +} + +type lifecycleKind uint8 + +const ( + lifecycleNormal lifecycleKind = iota + lifecyclePanic + lifecycleAborted + lifecycleTimeout +) + +func resolveTraceContext(r *http.Request) (traceID, spanID, parentSpanID string) { + if header := r.Header.Get(headerTraceparent); header != "" { + if t, parent, _, ok := trace.ParseTraceparent(header); ok { + return t, trace.NewSpanID(), parent + } + } + + traceID, _ = trace.NormalizeTraceID(r.Header.Get(headerTraceID)) + parentSpanID, _ = trace.NormalizeSpanID(r.Header.Get(headerSpanID)) + if traceID == "" { + traceID, _ = trace.NormalizeTraceID(r.Header.Get(headerRequestID)) + } + if traceID == "" { + traceID = trace.NewTraceID() + } + return traceID, trace.NewSpanID(), parentSpanID +} + +type statusWriter struct { + http.ResponseWriter + ctx context.Context + status int + wroteHeader bool +} + +func wrapResponseWriter(w http.ResponseWriter, ctx context.Context) *statusWriter { + return &statusWriter{ + ResponseWriter: w, + ctx: ctx, + status: http.StatusOK, + } +} + +func (w *statusWriter) Header() http.Header { + return w.ResponseWriter.Header() +} + +func (w *statusWriter) WriteHeader(statusCode int) { + w.status = statusCode + w.wroteHeader = true + waylogv2.SetHTTPStatus(w.ctx, statusCode) + w.ResponseWriter.WriteHeader(statusCode) +} + +func (w *statusWriter) Write(b []byte) (int, error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(b) +} + +func (w *statusWriter) ReadFrom(r io.Reader) (int64, error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok { + return rf.ReadFrom(r) + } + return io.Copy(w.ResponseWriter, r) +} + +func (w *statusWriter) Flush() { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + if flusher, ok := w.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +func (w *statusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := w.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return hijacker.Hijack() +} + +func (w *statusWriter) Push(target string, opts *http.PushOptions) error { + pusher, ok := w.ResponseWriter.(http.Pusher) + if !ok { + return http.ErrNotSupported + } + return pusher.Push(target, opts) +} + +func (w *statusWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +func (w *statusWriter) WroteHeader() bool { + return w.wroteHeader +} diff --git a/pkg/waylog/http/middleware_test.go b/pkg/waylog/http/middleware_test.go new file mode 100644 index 0000000..5c887d5 --- /dev/null +++ b/pkg/waylog/http/middleware_test.go @@ -0,0 +1,508 @@ +package wayloghttp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const schemaPath = "../../../docs/schema/v2.0.json" + +type harness struct { + t *testing.T + buf *bytes.Buffer +} + +func newHarness(t *testing.T, cfg waylogv2.Config) *harness { + t.Helper() + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if cfg.Service == "" { + cfg.Service = "checkout" + } + if cfg.Env == "" { + cfg.Env = "test" + } + cfg.Output = buf + if err := waylogv2.Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return &harness{t: t, buf: buf} +} + +func (h *harness) lastEvent() *eventv2.Event { + h.t.Helper() + out := h.buf.Bytes() + if len(out) == 0 { + return nil + } + lines := bytes.Split(bytes.TrimRight(out, "\n"), []byte{'\n'}) + var ev eventv2.Event + if err := json.Unmarshal(lines[len(lines)-1], &ev); err != nil { + h.t.Fatalf("unmarshal emitted event: %v", err) + } + return &ev +} + +func (h *harness) validateLast() *eventv2.Event { + h.t.Helper() + ev := h.lastEvent() + if ev == nil { + h.t.Fatal("no event emitted") + } + if err := eventv2.Validate(schemaPath, ev); err != nil { + raw, _ := json.MarshalIndent(ev, "", " ") + h.t.Fatalf("emitted event fails schema: %v\n%s", err, raw) + } + return ev +} + +func httpFields(t *testing.T, ev *eventv2.Event) map[string]any { + t.Helper() + fields, _ := ev.Fields["http"].(map[string]any) + if fields == nil { + t.Fatalf("fields.http missing: %+v", ev.Fields) + } + return fields +} + +func TestHTTPMiddlewareEmitsOK(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/health", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusOK { + t.Fatalf("status=%s want ok", ev.Status) + } + fields := httpFields(t, ev) + if got := fields["method"]; got != http.MethodGet { + t.Fatalf("method=%v want %s", got, http.MethodGet) + } + if got := fields["route"]; got != "/health" { + t.Fatalf("route=%v want /health", got) + } + if got, _ := fields["status"].(float64); got != 200 { + t.Fatalf("status=%v want 200", fields["status"]) + } +} + +func TestHTTPMiddlewareUsesPatternWhenPresent(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/checkout/123", nil) + req.Pattern = "/checkout/{id}" + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + fields := httpFields(t, ev) + if got := fields["route"]; got != "/checkout/{id}" { + t.Fatalf("route=%v want pattern", got) + } +} + +func TestHTTPMiddlewareWriteWithoutHeaderRecords200(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/write", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + fields := httpFields(t, ev) + if got, _ := fields["status"].(float64); got != 200 { + t.Fatalf("status=%v want 200", fields["status"]) + } +} + +func TestHTTPMiddlewarePreservesTraceparent(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/trace", nil) + req.Header.Set("traceparent", "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01") + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.TraceID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { + t.Fatalf("trace_id=%s want propagated trace id", ev.TraceID) + } + if ev.ParentSpanID != "bbbbbbbbbbbbbbbb" { + t.Fatalf("parent_span_id=%s want propagated parent span", ev.ParentSpanID) + } + if ev.SpanID == "" || ev.SpanID == ev.ParentSpanID { + t.Fatalf("span_id=%s want fresh local server span", ev.SpanID) + } +} + +func TestHTTPMiddlewareTraceFallbacks(t *testing.T) { + cases := []struct { + name string + headers map[string]string + wantTraceID string + wantParentSpan string + }{ + { + name: "x-trace-id and x-span-id", + headers: map[string]string{ + "x-trace-id": "cccccccccccccccccccccccccccccccc", + "x-span-id": "dddddddddddddddd", + }, + wantTraceID: "cccccccccccccccccccccccccccccccc", + wantParentSpan: "dddddddddddddddd", + }, + { + name: "x-request-id fallback", + headers: map[string]string{ + "x-request-id": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + }, + wantTraceID: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + }, + { + name: "fresh trace fallback", + headers: map[string]string{}, + wantTraceID: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/trace", nil) + for k, v := range tc.headers { + req.Header.Set(k, v) + } + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if tc.wantTraceID != "" && ev.TraceID != tc.wantTraceID { + t.Fatalf("trace_id=%s want %s", ev.TraceID, tc.wantTraceID) + } + if tc.wantTraceID == "" && ev.TraceID == "" { + t.Fatal("fresh trace fallback must generate a trace_id") + } + if tc.wantParentSpan != "" && ev.ParentSpanID != tc.wantParentSpan { + t.Fatalf("parent_span_id=%s want %s", ev.ParentSpanID, tc.wantParentSpan) + } + }) + } +} + +func TestHTTPMiddlewarePanicEmitsPanicAnchorAndWrites500(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + panic("boom") + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/panic", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != eventv2.CodePanic { + t.Fatalf("panic anchor wrong: %+v", ev.Anchor) + } + if rr.Code != http.StatusInternalServerError { + t.Fatalf("response code=%d want 500", rr.Code) + } + fields := httpFields(t, ev) + if got, _ := fields["status"].(float64); got != 500 { + t.Fatalf("fields.http.status=%v want 500", fields["status"]) + } +} + +func TestHTTPMiddlewarePanicAnchorsInnermostActiveStep(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = waylogv2.StepVoid(r.Context(), "db.load_cart", func(ctx context.Context) error { + panic("boom") + }) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/panic-step", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Anchor == nil || ev.Anchor.Step != "db.load_cart" || ev.Anchor.ErrorCode != eventv2.CodePanic { + t.Fatalf("panic anchor wrong: %+v", ev.Anchor) + } +} + +func TestHTTPMiddlewarePanicDoesNotOverwriteWrittenStatus(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + panic("boom") + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/panic-written", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + fields := httpFields(t, ev) + if got, _ := fields["status"].(float64); got != 204 { + t.Fatalf("fields.http.status=%v want 204", fields["status"]) + } + if rr.Code != http.StatusNoContent { + t.Fatalf("response code=%d want 204", rr.Code) + } +} + +func TestHTTPMiddlewareCancelEmitsAborted(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + reqCtx, cancel := context.WithCancel(context.Background()) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cancel() + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/abort", nil).WithContext(reqCtx) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusAborted { + t.Fatalf("status=%s want aborted", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != eventv2.CodeAborted { + t.Fatalf("aborted anchor wrong: %+v", ev.Anchor) + } +} + +func TestHTTPMiddlewareCancelKeepsExplicitFailure(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + reqCtx, cancel := context.WithCancel(context.Background()) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cancel() + waylogv2.Fail(r.Context(), waylogv2.NewError("AUTH_DENIED")) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/abort-fail", nil).WithContext(reqCtx) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.ErrorCode != "AUTH_DENIED" { + t.Fatalf("explicit failure must win over aborted finalize: %+v", ev.Anchor) + } +} + +func TestHTTPMiddlewareSuppressBeatsCancel(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + reqCtx, cancel := context.WithCancel(context.Background()) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + waylogv2.Suppress(r.Context()) + cancel() + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/suppress", nil).WithContext(reqCtx) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusSuppressed { + t.Fatalf("status=%s want suppressed", ev.Status) + } + if ev.Anchor != nil { + t.Fatalf("suppressed event must not carry anchor: %+v", ev.Anchor) + } +} + +func TestHTTPMiddlewareWatchdogEmitsTimeout(t *testing.T) { + h := newHarness(t, waylogv2.Config{MaxRequestAge: 20 * time.Millisecond}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(60 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/slow", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Status != eventv2.StatusTimeout { + t.Fatalf("status=%s want timeout", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != eventv2.CodeTimeout { + t.Fatalf("timeout anchor wrong: %+v", ev.Anchor) + } + if got := waylogv2.Stats().LateCompletionAfterEmit; got == 0 { + t.Fatalf("LateCompletionAfterEmit=%d want > 0 after timeout-first sealing", got) + } +} + +func TestHTTPMiddlewareWatchdogUsesInnermostActiveStep(t *testing.T) { + h := newHarness(t, waylogv2.Config{MaxRequestAge: 20 * time.Millisecond}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = waylogv2.StepVoid(r.Context(), "outer", func(ctx context.Context) error { + return waylogv2.StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + time.Sleep(60 * time.Millisecond) + return nil + }) + }) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/slow-step", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + if ev.Anchor == nil || ev.Anchor.Step != "payment.charge" || ev.Anchor.ErrorCode != eventv2.CodeTimeout { + t.Fatalf("timeout anchor wrong: %+v", ev.Anchor) + } +} + +func TestStatusWriterPreservesOptionalInterfaces(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + stub := newFancyWriter() + sw := wrapResponseWriter(stub, ctx) + + if _, ok := any(sw).(http.Flusher); !ok { + t.Fatal("wrapped writer must implement http.Flusher") + } + if _, ok := any(sw).(http.Hijacker); !ok { + t.Fatal("wrapped writer must implement http.Hijacker") + } + if _, ok := any(sw).(http.Pusher); !ok { + t.Fatal("wrapped writer must implement http.Pusher") + } + if _, ok := any(sw).(io.ReaderFrom); !ok { + t.Fatal("wrapped writer must implement io.ReaderFrom") + } + unwrapper, ok := any(sw).(interface{ Unwrap() http.ResponseWriter }) + if !ok || unwrapper.Unwrap() != stub { + t.Fatal("wrapped writer must expose Unwrap()") + } + + flusher := any(sw).(http.Flusher) + flusher.Flush() + if !stub.flushed { + t.Fatal("Flush was not delegated") + } + + pusher := any(sw).(http.Pusher) + if err := pusher.Push("/asset.js", nil); err != nil { + t.Fatalf("Push: %v", err) + } + if stub.pushedTarget != "/asset.js" { + t.Fatalf("push target=%q want /asset.js", stub.pushedTarget) + } + + hijacker := any(sw).(http.Hijacker) + conn, _, err := hijacker.Hijack() + if err != nil { + t.Fatalf("Hijack: %v", err) + } + _ = conn.Close() + if !stub.hijacked { + t.Fatal("Hijack was not delegated") + } + + readerFrom := any(sw).(io.ReaderFrom) + n, err := readerFrom.ReadFrom(strings.NewReader("hello")) + if err != nil { + t.Fatalf("ReadFrom: %v", err) + } + if n != 5 || !stub.readFromCalled { + t.Fatalf("ReadFrom delegation failed: n=%d called=%v", n, stub.readFromCalled) + } + if !sw.WroteHeader() || sw.status != http.StatusOK { + t.Fatalf("ReadFrom must auto-write 200: wrote=%v status=%d", sw.WroteHeader(), sw.status) + } + + _, _ = waylogv2.Finalize(ctx) + _ = h.validateLast() +} + +type fancyWriter struct { + header http.Header + status int + body bytes.Buffer + flushed bool + hijacked bool + pushedTarget string + readFromCalled bool +} + +func newFancyWriter() *fancyWriter { + return &fancyWriter{header: make(http.Header)} +} + +func (w *fancyWriter) Header() http.Header { + return w.header +} + +func (w *fancyWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.body.Write(p) +} + +func (w *fancyWriter) WriteHeader(statusCode int) { + w.status = statusCode +} + +func (w *fancyWriter) Flush() { + w.flushed = true +} + +func (w *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + w.hijacked = true + server, client := net.Pipe() + _ = client.Close() + rw := bufio.NewReadWriter(bufio.NewReader(server), bufio.NewWriter(server)) + return server, rw, nil +} + +func (w *fancyWriter) Push(target string, _ *http.PushOptions) error { + w.pushedTarget = target + return nil +} + +func (w *fancyWriter) ReadFrom(r io.Reader) (int64, error) { + w.readFromCalled = true + return io.Copy(&w.body, r) +} diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index e5f0d81..7a87797 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -11,6 +11,15 @@ import ( eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" ) +type lifecycleKind uint8 + +const ( + lifecycleNormal lifecycleKind = iota + lifecyclePanic + lifecycleAborted + lifecycleTimeout +) + // Finalize seals the request, assembles the v2.0 wide event, runs the // optional Redactor, writes the JSON line to Output, and removes the request // from the active set. Subsequent calls on the same context return (nil, nil) @@ -18,6 +27,26 @@ import ( // // Intended for middleware and adapter authors. func Finalize(ctx context.Context) (*eventv2.Event, error) { + return finalize(ctx, lifecycleNormal) +} + +// FinalizePanic seals the request as a panic-owned lifecycle emit. +func FinalizePanic(ctx context.Context) (*eventv2.Event, error) { + return finalize(ctx, lifecyclePanic) +} + +// FinalizeAborted seals the request as an aborted lifecycle emit unless the +// request had already failed explicitly, in which case the existing error wins. +func FinalizeAborted(ctx context.Context) (*eventv2.Event, error) { + return finalize(ctx, lifecycleAborted) +} + +// FinalizeTimeout seals the request as a timeout-owned lifecycle emit. +func FinalizeTimeout(ctx context.Context) (*eventv2.Event, error) { + return finalize(ctx, lifecycleTimeout) +} + +func finalize(ctx context.Context, lifecycle lifecycleKind) (*eventv2.Event, error) { r := requestFromContext(ctx) if r == nil { return nil, ErrNoActiveRequest @@ -29,6 +58,7 @@ func Finalize(ctx context.Context) (*eventv2.Event, error) { r.sdk.lateAfterEmit.Add(1) return nil, nil } + r.applyLifecycleLocked(lifecycle) r.sealed = true now := time.Now().UTC() ev := r.assembleLocked(now) @@ -52,6 +82,23 @@ func Finalize(ctx context.Context) (*eventv2.Event, error) { return ev, nil } +func (r *request) applyLifecycleLocked(lifecycle lifecycleKind) { + if r.suppressed { + return + } + + switch lifecycle { + case lifecyclePanic: + r.markLifecycleLocked(eventv2.StatusError, eventv2.CodePanic, "panic recovered") + case lifecycleTimeout: + r.markLifecycleLocked(eventv2.StatusTimeout, eventv2.CodeTimeout, "") + case lifecycleAborted: + if r.anchorStep == "" { + r.markLifecycleLocked(eventv2.StatusAborted, eventv2.CodeAborted, "") + } + } +} + func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { ev := &eventv2.Event{ SchemaVersion: eventv2.SchemaVersion2, @@ -140,6 +187,9 @@ func (r *request) snapshotStatusLocked() eventv2.Status { if r.suppressed { return eventv2.StatusSuppressed } + if r.finalStatus != "" { + return r.finalStatus + } if r.anchorStep != "" { return eventv2.StatusError } diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go index b52e850..e5eecf2 100644 --- a/pkg/waylog/v2/request.go +++ b/pkg/waylog/v2/request.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "sync" "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" ) const ( @@ -95,6 +97,21 @@ func SetField(ctx context.Context, key string, value any) { r.fields[key] = cloneDeep(value) } +// SetHTTPStatus updates fields.http.status on the active request. Intended for +// HTTP middleware and adapter authors. +func SetHTTPStatus(ctx context.Context, status int) { + r := requestFromContext(ctx) + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed || r.sealed { + return + } + r.setHTTPStatusLocked(status) +} + func cloneDeep(v any) any { switch x := v.(type) { case map[string]any: @@ -136,9 +153,10 @@ type request struct { bufBytes int - suppressed bool - sealed bool - headerOnly bool // degraded by MaxBufferBytes pressure (§4.4) + suppressed bool + sealed bool + headerOnly bool // degraded by MaxBufferBytes pressure (§4.4) + finalStatus eventv2.Status // First observable failing step, or "request" sentinel for a Fail() with // no active step. Set once and never overwritten. @@ -255,6 +273,32 @@ func (r *request) addLogLocked(l logBuf) { r.bufBytes += bytes } +func (r *request) activeStepLocked() string { + if n := len(r.stepStack); n > 0 { + return r.stepStack[n-1] + } + return "request" +} + +func (r *request) setHTTPStatusLocked(status int) { + httpMap, _ := r.fields["http"].(map[string]any) + if httpMap == nil { + httpMap = map[string]any{} + r.fields["http"] = httpMap + } + httpMap["status"] = status +} + +func (r *request) markLifecycleLocked(status eventv2.Status, code, reason string) { + if r.suppressed { + return + } + r.finalStatus = status + r.anchorStep = r.activeStepLocked() + r.anchorCode = code + r.recordErrorLocked(code, reason) +} + // degradeToHeaderOnlyLocked discards buffered detail and switches the request // to header-only emission. Anchor + fields + errs (already recorded) survive. func (r *request) degradeToHeaderOnlyLocked() { diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index c6e621d..8e6a4ee 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -232,6 +232,57 @@ func TestSuppressThenFailKeepsSuppressedAndCounts(t *testing.T) { } } +func TestFinalizePanicSynthesizesReservedLifecycleCode(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + r := requestFromContext(ctx) + r.pushStep("payment.charge") + + _, _ = FinalizePanic(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "payment.charge" || ev.Anchor.ErrorCode != eventv2.CodePanic { + t.Fatalf("panic anchor wrong: %+v", ev.Anchor) + } +} + +func TestFinalizeAbortedPreservesExistingExplicitFail(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + Fail(ctx, NewError("AUTH_DENIED")) + + _, _ = FinalizeAborted(ctx) + + h.validateLast() + ev := h.lastEvent() + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != "AUTH_DENIED" { + t.Fatalf("explicit failure must survive aborted finalize: %+v", ev.Anchor) + } +} + +func TestSetHTTPStatusUpdatesFieldsMap(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + SetField(ctx, "http", map[string]any{"method": "GET", "route": "/health", "status": 200}) + SetHTTPStatus(ctx, 502) + + _, _ = Finalize(ctx) + + h.validateLast() + ev := h.lastEvent() + httpMap, _ := ev.Fields["http"].(map[string]any) + if got, _ := httpMap["status"].(float64); got != 502 { + t.Fatalf("fields.http.status=%v want 502", httpMap["status"]) + } +} + func TestNewErrorRejectsReservedCodes(t *testing.T) { newHarness(t, Config{}) for _, code := range []string{eventv2.CodeTimeout, eventv2.CodeAborted, eventv2.CodePanic, eventv2.CodePartial} { diff --git a/pkg/waylog/v2/step.go b/pkg/waylog/v2/step.go index 2873082..76d8603 100644 --- a/pkg/waylog/v2/step.go +++ b/pkg/waylog/v2/step.go @@ -94,6 +94,7 @@ func Suppress(ctx context.Context) { r.errs = nil r.bufBytes = 0 r.headerOnly = false + r.finalStatus = "" r.anchorStep = "" r.anchorCode = "" } diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go index 157460e..ffd63a0 100644 --- a/pkg/waylog/v2/waylog.go +++ b/pkg/waylog/v2/waylog.go @@ -2,7 +2,8 @@ // // Public surface: Init, Shutdown, Stats, Logger, From, Step, StepVoid, Fail, // NewError, Suppress, Explain. Internal-but-exported helpers for middleware -// and adapter authors: Begin, Finalize, SetField. +// and adapter authors: Begin, Finalize, FinalizePanic, FinalizeAborted, +// FinalizeTimeout, SetField, SetHTTPStatus. package waylogv2 import ( @@ -190,6 +191,16 @@ func Stats() StatsSnapshot { } } +// MaxRequestAge returns the configured watchdog duration. Zero disables the +// HTTP middleware watchdog path. +func MaxRequestAge() time.Duration { + s := getState() + if s == nil { + return 0 + } + return s.cfg.MaxRequestAge +} + func getState() *sdk { stateMu.RLock() defer stateMu.RUnlock() From 98f932b5cf42a886ae87b82095180897c7e4f1c0 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 02:03:30 -0400 Subject: [PATCH 3/8] feat(sdk): added v2 outbound HTTP trace transport and downstream span linkage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds a new transport at pkg/waylog/http that: - injects W3C traceparent on outgoing HTTP requests - derives a fresh client span id for each downstream call - preserves the current request trace id while using the active request span as the parent context - records downstream linkage back onto the innermost active Waylog step The v2 core is extended so downstream call metadata survives request assembly: - replace the active step stack’s string-only shape with a structured active-step model - carry client step span_id through step close/finalization - carry structured downstream metadata onto emitted steps[] - export TraceID, SpanID, and RecordOutgoingSpan for adapter/transport authors - expose recorded downstream edges through Explain() and its text rendering Behavioral outcomes of this slice: - outgoing HTTP requests made inside a Step now emit a traceparent header - the matching completed step records steps[].span_id with the client span id sent downstream - the matching completed step records steps[].downstream with service/endpoint/kind metadata - Explain() now surfaces downstream edges instead of dropping that linkage on the floor Tests added and updated: - transport test proving traceparent injection - transport test proving step span_id + downstream metadata are recorded on the emitted event - transport test proving no active step means no recorded step linkage - core explain regression proving downstream edges appear in Explain() and String() --- pkg/waylog/http/transport.go | 44 +++++++++ pkg/waylog/http/transport_test.go | 146 ++++++++++++++++++++++++++++++ pkg/waylog/v2/assemble.go | 1 + pkg/waylog/v2/explain.go | 13 +++ pkg/waylog/v2/logger.go | 2 +- pkg/waylog/v2/request.go | 57 +++++++++++- pkg/waylog/v2/sdk_test.go | 25 +++++ pkg/waylog/v2/step.go | 21 +++-- pkg/waylog/v2/waylog.go | 2 +- 9 files changed, 299 insertions(+), 12 deletions(-) create mode 100644 pkg/waylog/http/transport.go create mode 100644 pkg/waylog/http/transport_test.go diff --git a/pkg/waylog/http/transport.go b/pkg/waylog/http/transport.go new file mode 100644 index 0000000..e365c0f --- /dev/null +++ b/pkg/waylog/http/transport.go @@ -0,0 +1,44 @@ +package wayloghttp + +import ( + "net/http" + + "github.com/sssmaran/WaylogCLI/pkg/trace" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +// Transport injects W3C traceparent on outgoing HTTP calls and records the +// client span on the innermost active Waylog step. +type Transport struct { + Base http.RoundTripper + DownstreamHint string +} + +// NewTransport returns a RoundTripper that propagates W3C trace context and +// records downstream linkage on the active step. +func NewTransport(base http.RoundTripper, downstreamHint string) *Transport { + if base == nil { + base = http.DefaultTransport + } + return &Transport{Base: base, DownstreamHint: downstreamHint} +} + +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + base := t.Base + if base == nil { + base = http.DefaultTransport + } + + traceID := waylogv2.TraceID(req.Context()) + parentSpan := waylogv2.SpanID(req.Context()) + if traceID == "" || parentSpan == "" { + return base.RoundTrip(req) + } + + clientSpan := trace.NewSpanID() + waylogv2.RecordOutgoingSpan(req.Context(), clientSpan, t.DownstreamHint, req.Method+" "+req.URL.Path) + + cloned := req.Clone(req.Context()) + cloned.Header.Set(headerTraceparent, trace.FormatTraceparent(traceID, clientSpan, "01")) + return base.RoundTrip(cloned) +} diff --git a/pkg/waylog/http/transport_test.go b/pkg/waylog/http/transport_test.go new file mode 100644 index 0000000..9ed1b6a --- /dev/null +++ b/pkg/waylog/http/transport_test.go @@ -0,0 +1,146 @@ +package wayloghttp + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +func TestRoundTripperInjectsTraceparent(t *testing.T) { + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if err := waylogv2.Init(waylogv2.Config{ + Service: "checkout", + Env: "test", + Output: buf, + }); err != nil { + t.Fatalf("Init: %v", err) + } + + var receivedTraceparent string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedTraceparent = r.Header.Get("traceparent") + w.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: "0123456789abcdef0123456789abcdef", + SpanID: "fedcba9876543210", + }) + + err := waylogv2.StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + client := &http.Client{Transport: NewTransport(http.DefaultTransport, "payment")} + req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstream.URL+"/charge", nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + return resp.Body.Close() + }) + if err != nil { + t.Fatalf("StepVoid: %v", err) + } + if _, err := waylogv2.Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + if receivedTraceparent == "" { + t.Fatal("expected traceparent injected") + } + wantPrefix := "00-0123456789abcdef0123456789abcdef-" + if len(receivedTraceparent) != 55 || receivedTraceparent[:len(wantPrefix)] != wantPrefix { + t.Fatalf("traceparent=%q want prefix %q and W3C shape", receivedTraceparent, wantPrefix) + } + + ev := lastHTTPTransportEvent(t, buf) + if len(ev.Steps) != 1 { + t.Fatalf("want 1 step, got %+v", ev.Steps) + } + step := ev.Steps[0] + if step.SpanID == "" { + t.Fatalf("step span_id missing: %+v", step) + } + if step.Downstream == nil { + t.Fatalf("downstream missing: %+v", step) + } + if step.Downstream.Service != "payment" || step.Downstream.Endpoint != "POST /charge" || step.Downstream.Kind != "rpc" { + t.Fatalf("downstream wrong: %+v", step.Downstream) + } +} + +func TestRoundTripperNoActiveStepStillInjectsButDoesNotRecordStepLinkage(t *testing.T) { + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if err := waylogv2.Init(waylogv2.Config{ + Service: "checkout", + Env: "test", + Output: buf, + }); err != nil { + t.Fatalf("Init: %v", err) + } + + var receivedTraceparent string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedTraceparent = r.Header.Get("traceparent") + w.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + SpanID: "bbbbbbbbbbbbbbbb", + }) + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, "payment")} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstream.URL+"/health", nil) + if err != nil { + t.Fatalf("NewRequestWithContext: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Do: %v", err) + } + _ = resp.Body.Close() + if _, err := waylogv2.Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + if receivedTraceparent == "" { + t.Fatal("expected traceparent injected") + } + ev := lastHTTPTransportEvent(t, buf) + if len(ev.Steps) != 0 { + t.Fatalf("no active step should mean no recorded step linkage: %+v", ev.Steps) + } +} + +func lastHTTPTransportEvent(t *testing.T, buf *bytes.Buffer) *eventv2.Event { + t.Helper() + raw := bytes.TrimSpace(buf.Bytes()) + if len(raw) == 0 { + t.Fatal("no event emitted") + } + lines := bytes.Split(raw, []byte{'\n'}) + var ev eventv2.Event + if err := json.Unmarshal(lines[len(lines)-1], &ev); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + return &ev +} diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index 7a87797..0ee6798 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -151,6 +151,7 @@ func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { StartMS: s.startMS, DurationMS: s.durationMS, Status: s.status, + Downstream: s.downstream, } if s.err != nil { step.Error = s.err.toStepError() diff --git a/pkg/waylog/v2/explain.go b/pkg/waylog/v2/explain.go index 361b3c2..cc32383 100644 --- a/pkg/waylog/v2/explain.go +++ b/pkg/waylog/v2/explain.go @@ -85,6 +85,13 @@ func Explain(ctx context.Context) (*ExplainResult, error) { si.ErrorMsg = s.err.Reason } out.Path = append(out.Path, si) + if s.downstream != nil { + out.Downstream = append(out.Downstream, DownstreamEdge{ + Step: s.name, + Service: s.downstream.Service, + Endpoint: s.downstream.Endpoint, + }) + } } for _, l := range r.logs { out.Logs = append(out.Logs, LogInfo{ @@ -120,6 +127,12 @@ func (r *ExplainResult) String() string { } fmt.Fprintf(&b, "[%s] [%dms] %s\n", l.Level, l.TsOffsetMS, l.Msg) } + if len(r.Downstream) > 0 { + b.WriteString("downstream:\n") + for _, d := range r.Downstream { + fmt.Fprintf(&b, " %s -> %s (%s)\n", d.Step, d.Service, d.Endpoint) + } + } return b.String() } diff --git a/pkg/waylog/v2/logger.go b/pkg/waylog/v2/logger.go index dd1a03e..03767db 100644 --- a/pkg/waylog/v2/logger.go +++ b/pkg/waylog/v2/logger.go @@ -64,7 +64,7 @@ func (r *request) appendLog(level, msg string, err *Error, more []F) { stepName := "" if n := len(r.stepStack); n > 0 { - stepName = r.stepStack[n-1] + stepName = r.stepStack[n-1].name } if err != nil { diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go index e5eecf2..3ffee33 100644 --- a/pkg/waylog/v2/request.go +++ b/pkg/waylog/v2/request.go @@ -112,6 +112,49 @@ func SetHTTPStatus(ctx context.Context, status int) { r.setHTTPStatusLocked(status) } +// TraceID returns the request trace id bound to ctx, or empty when absent. +func TraceID(ctx context.Context) string { + r := requestFromContext(ctx) + if r == nil { + return "" + } + r.mu.Lock() + defer r.mu.Unlock() + return r.traceID +} + +// SpanID returns the local server span id bound to ctx, or empty when absent. +func SpanID(ctx context.Context) string { + r := requestFromContext(ctx) + if r == nil { + return "" + } + r.mu.Lock() + defer r.mu.Unlock() + return r.spanID +} + +// RecordOutgoingSpan attaches a client span id and downstream edge to the +// innermost active step so the emitted event can be linked to child events. +func RecordOutgoingSpan(ctx context.Context, clientSpan, downstreamService, endpoint string) { + r := requestFromContext(ctx) + if r == nil || clientSpan == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed || r.sealed || len(r.stepStack) == 0 { + return + } + top := &r.stepStack[len(r.stepStack)-1] + top.spanID = clientSpan + top.downstream = &eventv2.Downstream{ + Service: downstreamService, + Endpoint: endpoint, + Kind: "rpc", + } +} + func cloneDeep(v any) any { switch x := v.(type) { case map[string]any: @@ -145,7 +188,7 @@ type request struct { mu sync.Mutex - stepStack []string + stepStack []activeStep steps []stepBuf logs []logBuf fields F @@ -164,12 +207,19 @@ type request struct { anchorCode string } +type activeStep struct { + name string + spanID string + downstream *eventv2.Downstream +} + type stepBuf struct { name string spanID string startMS int64 durationMS int64 status string + downstream *eventv2.Downstream err *Error } @@ -275,7 +325,7 @@ func (r *request) addLogLocked(l logBuf) { func (r *request) activeStepLocked() string { if n := len(r.stepStack); n > 0 { - return r.stepStack[n-1] + return r.stepStack[n-1].name } return "request" } @@ -435,6 +485,9 @@ func seedFromTime(b []byte) { // A stable upper bound is enough for the buffer-pressure cascade. func stepBytes(s stepBuf) int { n := 32 + len(s.name) + len(s.spanID) + if s.downstream != nil { + n += len(s.downstream.Service) + len(s.downstream.Endpoint) + len(s.downstream.Kind) + 24 + } if s.err != nil { n += 16 + len(s.err.Code) + len(s.err.Reason) + len(s.err.Cause) } diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index 8e6a4ee..5508461 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -603,6 +603,31 @@ func TestExplainSnapshotMidRequest(t *testing.T) { _, _ = Finalize(ctx) } +func TestExplainIncludesDownstreamEdges(t *testing.T) { + newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + _ = StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + RecordOutgoingSpan(ctx, "9d7a1b3e2c4d5e6f", "payment", "POST /charge") + return nil + }) + + res, err := Explain(ctx) + if err != nil { + t.Fatal(err) + } + if len(res.Downstream) != 1 { + t.Fatalf("want 1 downstream edge, got %+v", res.Downstream) + } + edge := res.Downstream[0] + if edge.Step != "payment.charge" || edge.Service != "payment" || edge.Endpoint != "POST /charge" { + t.Fatalf("downstream edge wrong: %+v", edge) + } + if !strings.Contains(res.String(), "payment.charge -> payment (POST /charge)") { + t.Fatalf("String() should include downstream edge: %s", res.String()) + } +} + func TestConcurrentLoggingDoesNotRace(t *testing.T) { newHarness(t, Config{MaxLogs: 100}) ctx := Begin(context.Background(), BeginOptions{}) diff --git a/pkg/waylog/v2/step.go b/pkg/waylog/v2/step.go index 76d8603..9d5c23b 100644 --- a/pkg/waylog/v2/step.go +++ b/pkg/waylog/v2/step.go @@ -67,7 +67,7 @@ func Fail(ctx context.Context, err *Error) { r.recordErrorLocked(err.Code, err.Reason) if r.anchorStep == "" { if n := len(r.stepStack); n > 0 { - r.anchorStep = r.stepStack[n-1] + r.anchorStep = r.stepStack[n-1].name } else { r.anchorStep = "request" } @@ -101,7 +101,7 @@ func Suppress(ctx context.Context) { func (r *request) pushStep(name string) { r.mu.Lock() - r.stepStack = append(r.stepStack, name) + r.stepStack = append(r.stepStack, activeStep{name: name}) r.mu.Unlock() } @@ -123,28 +123,33 @@ func (r *request) closeStep(name string, startMS, durationMS int64, fnErr error) r.mu.Lock() defer r.mu.Unlock() - r.popStackLocked(name) + active := r.popStackLocked(name) if r.suppressed || r.sealed { return } r.addStepLocked(stepBuf{ name: name, + spanID: active.spanID, startMS: startMS, durationMS: durationMS, status: status, + downstream: active.downstream, err: werr, }) } -func (r *request) popStackLocked(name string) { - if n := len(r.stepStack); n > 0 && r.stepStack[n-1] == name { +func (r *request) popStackLocked(name string) activeStep { + if n := len(r.stepStack); n > 0 && r.stepStack[n-1].name == name { + top := r.stepStack[n-1] r.stepStack = r.stepStack[:n-1] - return + return top } for i := len(r.stepStack) - 1; i >= 0; i-- { - if r.stepStack[i] == name { + if r.stepStack[i].name == name { + top := r.stepStack[i] r.stepStack = append(r.stepStack[:i], r.stepStack[i+1:]...) - return + return top } } + return activeStep{name: name} } diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go index ffd63a0..19e8dc1 100644 --- a/pkg/waylog/v2/waylog.go +++ b/pkg/waylog/v2/waylog.go @@ -3,7 +3,7 @@ // Public surface: Init, Shutdown, Stats, Logger, From, Step, StepVoid, Fail, // NewError, Suppress, Explain. Internal-but-exported helpers for middleware // and adapter authors: Begin, Finalize, FinalizePanic, FinalizeAborted, -// FinalizeTimeout, SetField, SetHTTPStatus. +// FinalizeTimeout, SetField, SetHTTPStatus, TraceID, SpanID, RecordOutgoingSpan. package waylogv2 import ( From 57964b2cddeb921e8a95cb35d92c27d90ac6e983 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 02:44:33 -0400 Subject: [PATCH 4/8] feat(sdk): add v2 delivery transport, dev dual-emit, and Go framework adapters Implemented the next Go v2 Phase 1 slices across transport delivery, local developer visibility, and framework expansion. Delivery transport: - add a new HTTP delivery client under pkg/transport/http - support direct single-event POSTs to /v1/events with application/json - add NDJSON batch flushing with application/x-ndjson - add two-class queueing for ok vs priority events - preserve the v2 pressure rule by evicting ok traffic before priority traffic - drain queued events best-effort during shutdown SDK wiring: - wire IngestURL and APIKey into pkg/waylog/v2 initialization - submit final events to the delivery client when IngestURL is configured - preserve writer-backed JSONL output as the default local-mode path when IngestURL is unset - keep emitted/suppressed counters aligned with accepted delivery rather than assuming every queue attempt succeeds Dev dual-emit: - enable local dev output when Config.DevMode is true or Env=dev - emit compact per-log pretty lines at log call time - emit pretty-printed final wide events at request completion - keep dev visibility orthogonal to the runtime truth path - serialize dev output behind a dedicated mutex so log-path emission and finalization do not race HTTP lifecycle reuse: - refactor pkg/waylog/http to expose a shared ServeHTTP core - keep net/http behavior unchanged while making panic/cancel/watchdog/finalize semantics reusable by framework adapters - add SetHTTPRoute to support frameworks that resolve route templates late Framework adapters: - add chi adapter with late route-template capture via chi RouteContext - add echo adapter that reuses the shared lifecycle while preserving echo route templates - add gin adapter with a framework-compatible response-writer bridge so handlers still see a gin.ResponseWriter while writes flow through the shared lifecycle wrapper Testing: - add transport tests for single-event POST, auth header propagation, NDJSON batching, multi-batch splitting, and queue priority preservation - add SDK tests for IngestURL transport routing, dev pretty-log output, dev final pretty JSON, and non-dev suppression of local pretty output - add conformance-style adapter tests for chi, echo, and gin route template capture and shared lifecycle behavior - keep the existing net/http and v2 core suites passing --- go.work.sum | 32 +++++ pkg/go.mod | 6 + pkg/transport/http/batch.go | 73 ++++++++++ pkg/transport/http/client.go | 125 +++++++++++++++++ pkg/transport/http/client_test.go | 146 +++++++++++++++++++ pkg/transport/http/queue.go | 216 +++++++++++++++++++++++++++++ pkg/waylog/chi/middleware.go | 27 ++++ pkg/waylog/chi/middleware_test.go | 93 +++++++++++++ pkg/waylog/echo/middleware.go | 26 ++++ pkg/waylog/echo/middleware_test.go | 90 ++++++++++++ pkg/waylog/gin/middleware.go | 125 +++++++++++++++++ pkg/waylog/gin/middleware_test.go | 94 +++++++++++++ pkg/waylog/http/middleware.go | 124 ++++++++++------- pkg/waylog/v2/assemble.go | 20 ++- pkg/waylog/v2/devemit.go | 74 ++++++++++ pkg/waylog/v2/errors.go | 7 +- pkg/waylog/v2/logger.go | 13 +- pkg/waylog/v2/request.go | 20 +++ pkg/waylog/v2/sdk_test.go | 116 +++++++++++++++- pkg/waylog/v2/waylog.go | 61 ++++++-- 20 files changed, 1413 insertions(+), 75 deletions(-) create mode 100644 pkg/transport/http/batch.go create mode 100644 pkg/transport/http/client.go create mode 100644 pkg/transport/http/client_test.go create mode 100644 pkg/transport/http/queue.go create mode 100644 pkg/waylog/chi/middleware.go create mode 100644 pkg/waylog/chi/middleware_test.go create mode 100644 pkg/waylog/echo/middleware.go create mode 100644 pkg/waylog/echo/middleware_test.go create mode 100644 pkg/waylog/gin/middleware.go create mode 100644 pkg/waylog/gin/middleware_test.go create mode 100644 pkg/waylog/v2/devemit.go diff --git a/go.work.sum b/go.work.sum index a3d98af..b3c5df5 100644 --- a/go.work.sum +++ b/go.work.sum @@ -6,17 +6,49 @@ github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= +github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= diff --git a/pkg/go.mod b/pkg/go.mod index abd30e1..a8e1ec7 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -3,3 +3,9 @@ module github.com/sssmaran/WaylogCLI/pkg go 1.24.2 require github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + +require ( + github.com/gin-gonic/gin v1.10.1 + github.com/go-chi/chi/v5 v5.2.3 + github.com/labstack/echo/v4 v4.13.4 +) diff --git a/pkg/transport/http/batch.go b/pkg/transport/http/batch.go new file mode 100644 index 0000000..93fb660 --- /dev/null +++ b/pkg/transport/http/batch.go @@ -0,0 +1,73 @@ +package transporthttp + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +func (c *Client) flushBatch(batch []*eventv2.Event) { + if c.url == "" || len(batch) == 0 { + return + } + + for _, chunk := range splitBatches(batch, c.cfg.MaxBatch, c.cfg.MaxBatchSize) { + var body bytes.Buffer + enc := json.NewEncoder(&body) + for _, ev := range chunk { + if err := enc.Encode(ev); err != nil { + continue + } + } + req, err := http.NewRequest(http.MethodPost, c.url, bytes.NewReader(body.Bytes())) + if err != nil { + continue + } + req.Header.Set("Content-Type", "application/x-ndjson") + if c.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) + } + resp, err := c.http.Do(req) + if err != nil { + continue + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } +} + +func splitBatches(batch []*eventv2.Event, maxCount, maxBytes int) [][]*eventv2.Event { + if len(batch) == 0 { + return nil + } + if maxCount <= 0 { + maxCount = len(batch) + } + if maxBytes <= 0 { + maxBytes = 1 << 20 + } + + var out [][]*eventv2.Event + start := 0 + for start < len(batch) { + end := start + size := 0 + for end < len(batch) && end-start < maxCount { + next := int(estimateEventSize(batch[end])) + if end > start && size+next > maxBytes { + break + } + size += next + end++ + } + if end == start { + end++ + } + out = append(out, batch[start:end]) + start = end + } + return out +} diff --git a/pkg/transport/http/client.go b/pkg/transport/http/client.go new file mode 100644 index 0000000..edadcca --- /dev/null +++ b/pkg/transport/http/client.go @@ -0,0 +1,125 @@ +package transporthttp + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +type Config struct { + IngestURL string + APIKey string + Timeout time.Duration + + BatchMode bool + MaxBatch int + MaxBatchSize int + BatchAgeMs int + OkBudgetPct int + InFlightCap int64 +} + +type Client struct { + cfg Config + url string + http *http.Client + queue *queue + closed sync.Once +} + +func New(cfg Config) *Client { + if cfg.Timeout <= 0 { + cfg.Timeout = 5 * time.Second + } + if cfg.MaxBatch <= 0 { + cfg.MaxBatch = 256 + } + if cfg.MaxBatchSize <= 0 { + cfg.MaxBatchSize = 1 << 20 + } + if cfg.BatchAgeMs <= 0 { + cfg.BatchAgeMs = 50 + } + if cfg.OkBudgetPct <= 0 { + cfg.OkBudgetPct = 70 + } + if cfg.InFlightCap <= 0 { + cfg.InFlightCap = 10 << 20 + } + + c := &Client{ + cfg: cfg, + url: normalizeIngestURL(cfg.IngestURL), + http: &http.Client{Timeout: cfg.Timeout}, + } + if cfg.BatchMode { + c.queue = newQueue(cfg, c.flushBatch) + go c.queue.run() + } + return c +} + +func normalizeIngestURL(raw string) string { + raw = strings.TrimRight(raw, "/") + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + if strings.HasSuffix(raw, "/v1/events") { + return raw + } + return raw + "/v1/events" +} + +func (c *Client) Submit(ev *eventv2.Event) bool { + if ev == nil || c.url == "" { + return false + } + if c.queue != nil { + return c.queue.enqueue(ev) + } + return c.submitSingle(ev) +} + +func (c *Client) Shutdown(timeout time.Duration) { + c.closed.Do(func() { + if c.queue != nil { + c.queue.shutdown(timeout) + } + }) +} + +func (c *Client) submitSingle(ev *eventv2.Event) bool { + if ev == nil || c.url == "" { + return false + } + raw, err := json.Marshal(ev) + if err != nil { + return false + } + req, err := http.NewRequest(http.MethodPost, c.url, bytes.NewReader(raw)) + if err != nil { + return false + } + req.Header.Set("Content-Type", "application/json") + if c.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) + } + resp, err := c.http.Do(req) + if err != nil { + return false + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300 +} diff --git a/pkg/transport/http/client_test.go b/pkg/transport/http/client_test.go new file mode 100644 index 0000000..e4fc8df --- /dev/null +++ b/pkg/transport/http/client_test.go @@ -0,0 +1,146 @@ +package transporthttp + +import ( + "bufio" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +func validEvent(id string, status eventv2.Status) *eventv2.Event { + return &eventv2.Event{ + SchemaVersion: eventv2.SchemaVersion2, + EventID: id, + TsStart: time.Unix(0, 0), + TsEnd: time.Unix(1, 0), + DurationMS: 1, + Kind: "http", + Service: "checkout", + Env: "test", + TraceID: "0123456789abcdef0123456789abcdef", + SpanID: "fedcba9876543210", + ParentSpanID: "", + Status: status, + } +} + +func TestClientSinglePost(t *testing.T) { + var got eventv2.Event + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/events" { + t.Fatalf("path=%s want /v1/events", r.URL.Path) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Fatalf("content-type=%s", r.Header.Get("Content-Type")) + } + if r.Header.Get("Authorization") != "Bearer k" { + t.Fatalf("authorization=%q want bearer", r.Header.Get("Authorization")) + } + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":1,"duplicate":0,"rejected":[]}`) + })) + defer srv.Close() + + cli := New(Config{IngestURL: srv.URL, APIKey: "k"}) + cli.Submit(validEvent("e1", eventv2.StatusOK)) + cli.Shutdown(2 * time.Second) + if got.EventID != "e1" { + t.Fatalf("server got %+v", got) + } +} + +func TestNDJSONBatchFlush(t *testing.T) { + var batches atomic.Int64 + var totalEvents atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-ndjson" { + t.Fatalf("content-type=%s", r.Header.Get("Content-Type")) + } + sc := bufio.NewScanner(r.Body) + n := int64(0) + for sc.Scan() { + var e eventv2.Event + if err := json.Unmarshal(sc.Bytes(), &e); err != nil { + t.Fatalf("decode: %v", err) + } + n++ + } + totalEvents.Add(n) + batches.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":1,"duplicate":0,"rejected":[]}`) + })) + defer srv.Close() + + cli := New(Config{IngestURL: srv.URL, BatchMode: true, BatchAgeMs: 20}) + for i := 0; i < 300; i++ { + cli.Submit(validEvent(itoa(i), eventv2.StatusOK)) + } + cli.Shutdown(3 * time.Second) + + if totalEvents.Load() != 300 { + t.Fatalf("total events=%d want 300", totalEvents.Load()) + } + if batches.Load() < 2 { + t.Fatalf("expected at least 2 batches, got %d", batches.Load()) + } +} + +func TestQueueEvictsOKBeforePriority(t *testing.T) { + var flushed []string + q := newQueue(Config{ + MaxBatch: 256, + MaxBatchSize: 1 << 20, + BatchAgeMs: 50, + OkBudgetPct: 70, + InFlightCap: 600, + }, func(batch []*eventv2.Event) { + for _, ev := range batch { + flushed = append(flushed, ev.EventID) + } + }) + go q.run() + + for i := 0; i < 6; i++ { + q.enqueue(validEvent("ok-"+itoa(i), eventv2.StatusOK)) + } + q.enqueue(validEvent("prio-1", eventv2.StatusError)) + q.shutdown(2 * time.Second) + + if len(flushed) == 0 { + t.Fatal("expected flushed events") + } + foundPriority := false + for _, id := range flushed { + if id == "prio-1" { + foundPriority = true + break + } + } + if !foundPriority { + t.Fatalf("priority event was evicted before ok events: %+v", flushed) + } +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + digits := [20]byte{} + n := len(digits) + for i > 0 { + n-- + digits[n] = byte('0' + i%10) + i /= 10 + } + return string(digits[n:]) +} diff --git a/pkg/transport/http/queue.go b/pkg/transport/http/queue.go new file mode 100644 index 0000000..c7bbd8c --- /dev/null +++ b/pkg/transport/http/queue.go @@ -0,0 +1,216 @@ +package transporthttp + +import ( + "bytes" + "encoding/json" + "sync" + "time" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +type queuedEvent struct { + ev *eventv2.Event + size int64 +} + +type queue struct { + cfg Config + flushFn func(batch []*eventv2.Event) + + mu sync.Mutex + okQ []queuedEvent + prioQ []queuedEvent + okBytes int64 + prioBytes int64 + closing bool + notify chan struct{} + stop chan struct{} + drained chan struct{} +} + +func newQueue(cfg Config, flushFn func(batch []*eventv2.Event)) *queue { + return &queue{ + cfg: cfg, + flushFn: flushFn, + notify: make(chan struct{}, 1), + stop: make(chan struct{}), + drained: make(chan struct{}), + } +} + +func (q *queue) enqueue(ev *eventv2.Event) bool { + if ev == nil { + return false + } + size := estimateEventSize(ev) + + q.mu.Lock() + defer q.mu.Unlock() + if q.closing { + return false + } + + accepted := false + switch ev.Status { + case eventv2.StatusOK, eventv2.StatusSuppressed: + accepted = q.enqueueOKLocked(ev, size) + default: + accepted = q.enqueuePriorityLocked(ev, size) + } + if accepted { + q.signal() + } + return accepted +} + +func (q *queue) enqueueOKLocked(ev *eventv2.Event, size int64) bool { + okCap := q.cfg.InFlightCap * int64(q.cfg.OkBudgetPct) / 100 + if size > okCap { + return false + } + for q.okBytes+size > okCap && len(q.okQ) > 0 { + q.okBytes -= q.okQ[0].size + q.okQ = q.okQ[1:] + } + if q.okBytes+size > okCap { + return false + } + q.okQ = append(q.okQ, queuedEvent{ev: ev, size: size}) + q.okBytes += size + return true +} + +func (q *queue) enqueuePriorityLocked(ev *eventv2.Event, size int64) bool { + totalCap := q.cfg.InFlightCap + if size > totalCap { + return false + } + for q.totalBytesLocked()+size > totalCap && len(q.okQ) > 0 { + q.okBytes -= q.okQ[0].size + q.okQ = q.okQ[1:] + } + for q.totalBytesLocked()+size > totalCap && len(q.prioQ) > 0 { + q.prioBytes -= q.prioQ[0].size + q.prioQ = q.prioQ[1:] + } + if q.totalBytesLocked()+size > totalCap { + return false + } + q.prioQ = append(q.prioQ, queuedEvent{ev: ev, size: size}) + q.prioBytes += size + return true +} + +func (q *queue) totalBytesLocked() int64 { + return q.okBytes + q.prioBytes +} + +func (q *queue) run() { + ticker := time.NewTicker(time.Duration(q.cfg.BatchAgeMs) * time.Millisecond) + defer ticker.Stop() + defer close(q.drained) + + for { + select { + case <-q.notify: + for _, batch := range q.takeReadyBatches(false) { + q.flushFn(batch) + } + case <-ticker.C: + for _, batch := range q.takeReadyBatches(true) { + q.flushFn(batch) + } + case <-q.stop: + for _, batch := range q.takeReadyBatches(true) { + q.flushFn(batch) + } + return + } + } +} + +func (q *queue) takeReadyBatches(force bool) [][]*eventv2.Event { + q.mu.Lock() + defer q.mu.Unlock() + + var out [][]*eventv2.Event + for { + batch := q.takeOneLocked(force) + if len(batch) == 0 { + return out + } + out = append(out, batch) + } +} + +func (q *queue) takeOneLocked(force bool) []*eventv2.Event { + if batch := q.takeFromLocked(&q.prioQ, &q.prioBytes, force); len(batch) > 0 { + return batch + } + if batch := q.takeFromLocked(&q.okQ, &q.okBytes, force); len(batch) > 0 { + return batch + } + return nil +} + +func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool) []*eventv2.Event { + if len(*src) == 0 { + return nil + } + if !force && len(*src) < q.cfg.MaxBatch && *totalBytes < int64(q.cfg.MaxBatchSize) { + return nil + } + + var batch []*eventv2.Event + var batchBytes int64 + n := 0 + for n < len(*src) && n < q.cfg.MaxBatch { + next := (*src)[n] + if n > 0 && batchBytes+next.size > int64(q.cfg.MaxBatchSize) { + break + } + batch = append(batch, next.ev) + batchBytes += next.size + n++ + } + *src = (*src)[n:] + *totalBytes -= batchBytes + return batch +} + +func (q *queue) shutdown(timeout time.Duration) { + q.mu.Lock() + if q.closing { + q.mu.Unlock() + } else { + q.closing = true + close(q.stop) + q.mu.Unlock() + } + + if timeout <= 0 { + <-q.drained + return + } + + select { + case <-q.drained: + case <-time.After(timeout): + } +} + +func (q *queue) signal() { + select { + case q.notify <- struct{}{}: + default: + } +} + +func estimateEventSize(ev *eventv2.Event) int64 { + raw, err := json.Marshal(ev) + if err != nil { + return 0 + } + return int64(len(bytes.TrimSpace(raw)) + 1) +} diff --git a/pkg/waylog/chi/middleware.go b/pkg/waylog/chi/middleware.go new file mode 100644 index 0000000..b986cde --- /dev/null +++ b/pkg/waylog/chi/middleware.go @@ -0,0 +1,27 @@ +package waylogchi + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +// Middleware applies the shared Waylog v2 HTTP lifecycle to chi handlers while +// preserving the router's route template when available. +func Middleware(next http.Handler) http.Handler { + if next == nil { + next = http.NotFoundHandler() + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route := "" + wayloghttp.ServeHTTP(w, r, route, func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + if rc := chi.RouteContext(r.Context()); rc != nil { + waylogv2.SetHTTPRoute(r.Context(), rc.RoutePattern()) + } + }) + }) +} diff --git a/pkg/waylog/chi/middleware_test.go b/pkg/waylog/chi/middleware_test.go new file mode 100644 index 0000000..0a09337 --- /dev/null +++ b/pkg/waylog/chi/middleware_test.go @@ -0,0 +1,93 @@ +package waylogchi + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const schemaPath = "../../../docs/schema/v2.0.json" + +func newHarness(t *testing.T, cfg waylogv2.Config) *bytes.Buffer { + t.Helper() + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if cfg.Service == "" { + cfg.Service = "checkout" + } + if cfg.Env == "" { + cfg.Env = "test" + } + cfg.Output = buf + if err := waylogv2.Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return buf +} + +func lastEvent(t *testing.T, buf *bytes.Buffer) *eventv2.Event { + t.Helper() + var ev eventv2.Event + if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &ev); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + if err := eventv2.Validate(schemaPath, &ev); err != nil { + t.Fatalf("schema validate: %v", err) + } + return &ev +} + +func TestMiddlewareCapturesRouteTemplate(t *testing.T) { + buf := newHarness(t, waylogv2.Config{}) + + r := chi.NewRouter() + r.Use(Middleware) + r.Get("/orders/{id}", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/orders/42", nil) + r.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + httpFields, _ := ev.Fields["http"].(map[string]any) + if got := httpFields["route"]; got != "/orders/{id}" { + t.Fatalf("route=%v want /orders/{id}", got) + } +} + +func TestMiddlewareTimeoutUsesSharedLifecycle(t *testing.T) { + buf := newHarness(t, waylogv2.Config{MaxRequestAge: 10 * time.Millisecond}) + + r := chi.NewRouter() + r.Use(Middleware) + r.Get("/slow/{id}", func(w http.ResponseWriter, r *http.Request) { + time.Sleep(30 * time.Millisecond) + w.WriteHeader(http.StatusOK) + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/slow/42", nil) + r.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + if ev.Status != eventv2.StatusTimeout { + t.Fatalf("status=%s want timeout", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.ErrorCode != eventv2.CodeTimeout { + t.Fatalf("timeout anchor wrong: %+v", ev.Anchor) + } +} diff --git a/pkg/waylog/echo/middleware.go b/pkg/waylog/echo/middleware.go new file mode 100644 index 0000000..adf289a --- /dev/null +++ b/pkg/waylog/echo/middleware.go @@ -0,0 +1,26 @@ +package waylogecho + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" +) + +// Middleware applies the shared Waylog v2 HTTP lifecycle to Echo handlers +// while preserving Echo's route template in fields.http.route. +func Middleware() echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + wayloghttp.ServeHTTP(c.Response().Writer, c.Request(), c.Path(), func(w http.ResponseWriter, r *http.Request) { + c.SetRequest(r) + c.Response().Writer = w + if err := next(c); err != nil { + c.Error(err) + } + }) + return nil + } + } +} diff --git a/pkg/waylog/echo/middleware_test.go b/pkg/waylog/echo/middleware_test.go new file mode 100644 index 0000000..ed9fcee --- /dev/null +++ b/pkg/waylog/echo/middleware_test.go @@ -0,0 +1,90 @@ +package waylogecho + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/labstack/echo/v4" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const schemaPath = "../../../docs/schema/v2.0.json" + +func newHarness(t *testing.T, cfg waylogv2.Config) *bytes.Buffer { + t.Helper() + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if cfg.Service == "" { + cfg.Service = "checkout" + } + if cfg.Env == "" { + cfg.Env = "test" + } + cfg.Output = buf + if err := waylogv2.Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return buf +} + +func lastEvent(t *testing.T, buf *bytes.Buffer) *eventv2.Event { + t.Helper() + var ev eventv2.Event + if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &ev); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + if err := eventv2.Validate(schemaPath, &ev); err != nil { + t.Fatalf("schema validate: %v", err) + } + return &ev +} + +func TestMiddlewareCapturesRouteTemplate(t *testing.T) { + buf := newHarness(t, waylogv2.Config{}) + + e := echo.New() + e.Use(Middleware()) + e.GET("/orders/:id", func(c echo.Context) error { + return c.NoContent(http.StatusNoContent) + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/orders/42", nil) + e.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + httpFields, _ := ev.Fields["http"].(map[string]any) + if got := httpFields["route"]; got != "/orders/:id" { + t.Fatalf("route=%v want /orders/:id", got) + } +} + +func TestMiddlewareKeepsSuppressedStatus(t *testing.T) { + buf := newHarness(t, waylogv2.Config{}) + + e := echo.New() + e.Use(Middleware()) + e.GET("/quiet/:id", func(c echo.Context) error { + waylogv2.Suppress(c.Request().Context()) + return c.NoContent(http.StatusNoContent) + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/quiet/42", nil) + e.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + if ev.Status != eventv2.StatusSuppressed { + t.Fatalf("status=%s want suppressed", ev.Status) + } +} diff --git a/pkg/waylog/gin/middleware.go b/pkg/waylog/gin/middleware.go new file mode 100644 index 0000000..8cbfd7f --- /dev/null +++ b/pkg/waylog/gin/middleware.go @@ -0,0 +1,125 @@ +package wayloggin + +import ( + "bufio" + "io" + "net" + "net/http" + + "github.com/gin-gonic/gin" + + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" +) + +// Middleware applies the shared Waylog v2 HTTP lifecycle to Gin handlers +// while preserving Gin's route template in fields.http.route. +func Middleware() gin.HandlerFunc { + return func(c *gin.Context) { + wayloghttp.ServeHTTP(c.Writer, c.Request, c.FullPath(), func(w http.ResponseWriter, r *http.Request) { + c.Request = r + c.Writer = newResponseWriter(w) + c.Next() + }) + } +} + +type responseWriter struct { + http.ResponseWriter + status int + size int +} + +func newResponseWriter(w http.ResponseWriter) *responseWriter { + return &responseWriter{ + ResponseWriter: w, + status: http.StatusOK, + size: -1, + } +} + +func (w *responseWriter) WriteHeader(code int) { + if w.Written() { + return + } + if code > 0 { + w.status = code + } + w.ResponseWriter.WriteHeader(code) + if w.size < 0 { + w.size = 0 + } +} + +func (w *responseWriter) WriteHeaderNow() { + if !w.Written() { + w.WriteHeader(w.status) + } +} + +func (w *responseWriter) Write(data []byte) (int, error) { + if !w.Written() { + w.WriteHeader(w.status) + } + n, err := w.ResponseWriter.Write(data) + if w.size < 0 { + w.size = 0 + } + w.size += n + return n, err +} + +func (w *responseWriter) WriteString(s string) (int, error) { + if !w.Written() { + w.WriteHeader(w.status) + } + n, err := io.WriteString(w.ResponseWriter, s) + if w.size < 0 { + w.size = 0 + } + w.size += n + return n, err +} + +func (w *responseWriter) Status() int { + return w.status +} + +func (w *responseWriter) Size() int { + return w.size +} + +func (w *responseWriter) Written() bool { + return w.size >= 0 +} + +func (w *responseWriter) Pusher() http.Pusher { + if pusher, ok := w.ResponseWriter.(http.Pusher); ok { + return pusher + } + return nil +} + +func (w *responseWriter) Flush() { + w.WriteHeaderNow() + if flusher, ok := w.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := w.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return hijacker.Hijack() +} + +func (w *responseWriter) CloseNotify() <-chan bool { + closeNotifier, ok := w.ResponseWriter.(http.CloseNotifier) + if !ok { + ch := make(chan bool) + close(ch) + return ch + } + return closeNotifier.CloseNotify() +} diff --git a/pkg/waylog/gin/middleware_test.go b/pkg/waylog/gin/middleware_test.go new file mode 100644 index 0000000..f85909b --- /dev/null +++ b/pkg/waylog/gin/middleware_test.go @@ -0,0 +1,94 @@ +package wayloggin + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const schemaPath = "../../../docs/schema/v2.0.json" + +func newHarness(t *testing.T, cfg waylogv2.Config) *bytes.Buffer { + t.Helper() + deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = waylogv2.Shutdown(deadline) + + buf := &bytes.Buffer{} + if cfg.Service == "" { + cfg.Service = "checkout" + } + if cfg.Env == "" { + cfg.Env = "test" + } + cfg.Output = buf + if err := waylogv2.Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return buf +} + +func lastEvent(t *testing.T, buf *bytes.Buffer) *eventv2.Event { + t.Helper() + var ev eventv2.Event + if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &ev); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + if err := eventv2.Validate(schemaPath, &ev); err != nil { + t.Fatalf("schema validate: %v", err) + } + return &ev +} + +func TestMiddlewareCapturesRouteTemplate(t *testing.T) { + gin.SetMode(gin.TestMode) + buf := newHarness(t, waylogv2.Config{}) + + r := gin.New() + r.Use(Middleware()) + r.GET("/orders/:id", func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/orders/42", nil) + r.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + httpFields, _ := ev.Fields["http"].(map[string]any) + if got := httpFields["route"]; got != "/orders/:id" { + t.Fatalf("route=%v want /orders/:id", got) + } +} + +func TestMiddlewarePanicUsesSharedLifecycle(t *testing.T) { + gin.SetMode(gin.TestMode) + buf := newHarness(t, waylogv2.Config{}) + + r := gin.New() + r.Use(Middleware()) + r.GET("/panic/:id", func(c *gin.Context) { + panic("boom") + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/panic/42", nil) + r.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.ErrorCode != eventv2.CodePanic { + t.Fatalf("panic anchor wrong: %+v", ev.Anchor) + } +} diff --git a/pkg/waylog/http/middleware.go b/pkg/waylog/http/middleware.go index 9eb8eb4..8d3cd17 100644 --- a/pkg/waylog/http/middleware.go +++ b/pkg/waylog/http/middleware.go @@ -28,68 +28,78 @@ func HTTP(next http.Handler) http.Handler { } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - traceID, spanID, parentSpanID := resolveTraceContext(r) - ctx := waylogv2.Begin(r.Context(), waylogv2.BeginOptions{ - TraceID: traceID, - SpanID: spanID, - ParentSpanID: parentSpanID, - }) + ServeHTTP(w, r, routePattern(r), next.ServeHTTP) + }) +} - route := r.Pattern - if route == "" { - route = r.URL.Path - } - waylogv2.SetField(ctx, "http", waylogv2.F{ - "method": r.Method, - "route": route, - "status": http.StatusOK, - }) +// ServeHTTP applies the shared Waylog v2 HTTP lifecycle to a request using +// the provided route template and next callback. Framework adapters should +// call this rather than reimplementing panic/cancel/watchdog precedence. +func ServeHTTP(w http.ResponseWriter, r *http.Request, route string, next func(http.ResponseWriter, *http.Request)) { + if next == nil { + next = http.NotFoundHandler().ServeHTTP + } - sw := wrapResponseWriter(w, ctx) - var sealed atomic.Bool + traceID, spanID, parentSpanID := resolveTraceContext(r) + ctx := waylogv2.Begin(r.Context(), waylogv2.BeginOptions{ + TraceID: traceID, + SpanID: spanID, + ParentSpanID: parentSpanID, + }) - deliver := func(kind lifecycleKind) { - if !sealed.CompareAndSwap(false, true) { - _, _ = waylogv2.Finalize(ctx) - return - } - switch kind { - case lifecycleTimeout: - _, _ = waylogv2.FinalizeTimeout(ctx) - case lifecyclePanic: - _, _ = waylogv2.FinalizePanic(ctx) - case lifecycleAborted: - _, _ = waylogv2.FinalizeAborted(ctx) - default: - _, _ = waylogv2.Finalize(ctx) - } - } + if route == "" { + route = routePattern(r) + } + waylogv2.SetField(ctx, "http", waylogv2.F{ + "method": r.Method, + "route": route, + "status": http.StatusOK, + }) + + sw := wrapResponseWriter(w, ctx) + var sealed atomic.Bool - if d := waylogv2.MaxRequestAge(); d > 0 { - timer := time.AfterFunc(d, func() { - deliver(lifecycleTimeout) - }) - defer timer.Stop() + deliver := func(kind lifecycleKind) { + if !sealed.CompareAndSwap(false, true) { + _, _ = waylogv2.Finalize(ctx) + return } + switch kind { + case lifecycleTimeout: + _, _ = waylogv2.FinalizeTimeout(ctx) + case lifecyclePanic: + _, _ = waylogv2.FinalizePanic(ctx) + case lifecycleAborted: + _, _ = waylogv2.FinalizeAborted(ctx) + default: + _, _ = waylogv2.Finalize(ctx) + } + } - defer func() { - if rec := recover(); rec != nil { - if !sw.WroteHeader() { - sw.WriteHeader(http.StatusInternalServerError) - } - deliver(lifecyclePanic) + if d := waylogv2.MaxRequestAge(); d > 0 { + timer := time.AfterFunc(d, func() { + deliver(lifecycleTimeout) + }) + defer timer.Stop() + } + + defer func() { + if rec := recover(); rec != nil { + if !sw.WroteHeader() { + sw.WriteHeader(http.StatusInternalServerError) } - }() + deliver(lifecyclePanic) + } + }() - next.ServeHTTP(sw, r.WithContext(ctx)) + next(sw, r.WithContext(ctx)) - if ctx.Err() != nil { - deliver(lifecycleAborted) - return - } + if ctx.Err() != nil { + deliver(lifecycleAborted) + return + } - deliver(lifecycleNormal) - }) + deliver(lifecycleNormal) } type lifecycleKind uint8 @@ -119,6 +129,16 @@ func resolveTraceContext(r *http.Request) (traceID, spanID, parentSpanID string) return traceID, trace.NewSpanID(), parentSpanID } +func routePattern(r *http.Request) string { + if r == nil { + return "" + } + if r.Pattern != "" { + return r.Pattern + } + return r.URL.Path +} + type statusWriter struct { http.ResponseWriter ctx context.Context diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index 0ee6798..f1ff947 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -72,13 +72,17 @@ func finalize(ctx context.Context, lifecycle lifecycleKind) (*eventv2.Event, err ev.Fields = r.sdk.cfg.Redactor(ev.Fields) } - if err := emit(r.sdk.out, ev); err != nil { + accepted, err := deliver(r.sdk, ev) + if err != nil { return ev, err } - r.sdk.emitted.Add(1) - if ev.Status == eventv2.StatusSuppressed { + if accepted { + r.sdk.emitted.Add(1) + } + if accepted && ev.Status == eventv2.StatusSuppressed { r.sdk.suppressed.Add(1) } + r.sdk.emitDevFinal(ev) return ev, nil } @@ -208,3 +212,13 @@ func emit(w io.Writer, ev *eventv2.Event) error { } return nil } + +func deliver(s *sdk, ev *eventv2.Event) (bool, error) { + if s.delivery != nil { + return s.delivery.Submit(ev), nil + } + if err := emit(s.out, ev); err != nil { + return false, err + } + return true, nil +} diff --git a/pkg/waylog/v2/devemit.go b/pkg/waylog/v2/devemit.go new file mode 100644 index 0000000..cfc9d62 --- /dev/null +++ b/pkg/waylog/v2/devemit.go @@ -0,0 +1,74 @@ +package waylogv2 + +import ( + "encoding/json" + "fmt" + "io" + "slices" + "strings" + + eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" +) + +type devLogRecord struct { + service string + traceID string + level string + msg string + step string + fields F +} + +func (s *sdk) emitDevLog(rec devLogRecord) { + if !s.devEnabled || s.devOut == nil { + return + } + + traceID := rec.traceID + if len(traceID) > 8 { + traceID = traceID[:8] + } + + parts := []string{ + fmt.Sprintf("[%s]", strings.ToUpper(rec.level)), + rec.service, + traceID, + } + if rec.step != "" { + parts = append(parts, rec.step) + } + parts = append(parts, rec.msg) + + for _, k := range sortedKeys(rec.fields) { + parts = append(parts, fmt.Sprintf("%s=%v", k, rec.fields[k])) + } + + s.devMu.Lock() + defer s.devMu.Unlock() + _, _ = io.WriteString(s.devOut, strings.Join(parts, " ")+"\n") +} + +func (s *sdk) emitDevFinal(ev *eventv2.Event) { + if !s.devEnabled || s.devOut == nil || ev == nil { + return + } + raw, err := json.MarshalIndent(ev, "", " ") + if err != nil { + return + } + s.devMu.Lock() + defer s.devMu.Unlock() + _, _ = s.devOut.Write(append(raw, '\n')) +} + +func sortedKeys(m F) []string { + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} diff --git a/pkg/waylog/v2/errors.go b/pkg/waylog/v2/errors.go index b56a92e..9a710e2 100644 --- a/pkg/waylog/v2/errors.go +++ b/pkg/waylog/v2/errors.go @@ -2,7 +2,6 @@ package waylogv2 import ( "fmt" - "os" eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" ) @@ -59,8 +58,10 @@ func recordReservedRejection(code, where string) { return } s.reservedRejected.Add(1) - if s.cfg.DevMode { - fmt.Fprintf(os.Stderr, "waylog: %s rejected reserved error code %q\n", where, code) + if s.devEnabled && s.devOut != nil { + s.devMu.Lock() + defer s.devMu.Unlock() + fmt.Fprintf(s.devOut, "waylog: %s rejected reserved error code %q\n", where, code) } } diff --git a/pkg/waylog/v2/logger.go b/pkg/waylog/v2/logger.go index 03767db..bb82691 100644 --- a/pkg/waylog/v2/logger.go +++ b/pkg/waylog/v2/logger.go @@ -57,8 +57,8 @@ func (r *request) appendLog(level, msg string, err *Error, more []F) { } r.mu.Lock() - defer r.mu.Unlock() if r.suppressed || r.sealed { + r.mu.Unlock() return } @@ -78,6 +78,17 @@ func (r *request) appendLog(level, msg string, err *Error, more []F) { fields: merged, stepName: stepName, }) + + dev := devLogRecord{ + service: r.sdk.cfg.Service, + traceID: r.traceID, + level: level, + msg: msg, + step: stepName, + fields: merged, + } + r.mu.Unlock() + r.sdk.emitDevLog(dev) } // mergeFields returns a freshly-allocated shallow copy of the merged maps. diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go index 3ffee33..4aac97d 100644 --- a/pkg/waylog/v2/request.go +++ b/pkg/waylog/v2/request.go @@ -112,6 +112,26 @@ func SetHTTPStatus(ctx context.Context, status int) { r.setHTTPStatusLocked(status) } +// SetHTTPRoute updates fields.http.route on the active request. Intended for +// HTTP middleware and adapter authors that resolve route templates late. +func SetHTTPRoute(ctx context.Context, route string) { + r := requestFromContext(ctx) + if r == nil || route == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.suppressed || r.sealed { + return + } + httpFields, _ := r.fields["http"].(map[string]any) + if httpFields == nil { + httpFields = map[string]any{} + r.fields["http"] = httpFields + } + httpFields["route"] = route +} + // TraceID returns the request trace id bound to ctx, or empty when absent. func TraceID(ctx context.Context) string { r := requestFromContext(ctx) diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index 5508461..fd07519 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "errors" + "net/http" + "net/http/httptest" "strings" "sync" "testing" @@ -16,8 +18,9 @@ import ( const schemaPath = "../../../docs/schema/v2.0.json" type harness struct { - t *testing.T - buf *bytes.Buffer + t *testing.T + buf *bytes.Buffer + devBuf *bytes.Buffer } func newHarness(t *testing.T, cfg Config) *harness { @@ -34,7 +37,7 @@ func newHarness(t *testing.T, cfg Config) *harness { if err := Init(cfg); err != nil { t.Fatalf("Init: %v", err) } - return &harness{t: t, buf: buf} + return &harness{t: t, buf: buf, devBuf: &bytes.Buffer{}} } func (h *harness) lastEvent() *eventv2.Event { @@ -64,6 +67,15 @@ func (h *harness) validateLast() { } } +func (h *harness) captureDevOutput() { + h.t.Helper() + s := getState() + if s == nil { + h.t.Fatal("sdk not initialized") + } + s.devOut = h.devBuf +} + func TestInitRequiresServiceAndEnv(t *testing.T) { resetForTest() if err := Init(Config{}); err == nil { @@ -725,3 +737,101 @@ func TestShutdownTimesOutWithStuckRequest(t *testing.T) { t.Fatal("expected timeout error") } } + +func TestIngestURLRoutesFinalEventToTransport(t *testing.T) { + var ( + mu sync.Mutex + got *eventv2.Event + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ev eventv2.Event + if err := json.NewDecoder(r.Body).Decode(&ev); err != nil { + t.Fatalf("decode ingest payload: %v", err) + } + mu.Lock() + got = &ev + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + h := newHarness(t, Config{IngestURL: srv.URL}) + ctx := Begin(context.Background(), BeginOptions{}) + SetField(ctx, "http", map[string]any{"method": "GET", "route": "/health", "status": 200}) + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + deadline, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := Shutdown(deadline); err != nil { + t.Fatalf("Shutdown: %v", err) + } + + if h.buf.Len() != 0 { + t.Fatalf("local writer should stay empty when IngestURL is set, got %q", h.buf.String()) + } + mu.Lock() + defer mu.Unlock() + if got == nil || got.TraceID == "" { + t.Fatalf("transport did not receive emitted event: %+v", got) + } +} + +func TestDevModeEmitsPrettyLogsAndFinalEvent(t *testing.T) { + h := newHarness(t, Config{DevMode: true}) + h.captureDevOutput() + ctx := Begin(context.Background(), BeginOptions{}) + + From(ctx).Info("served", F{"route": "/x"}) + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + out := h.devBuf.String() + if !strings.Contains(out, "[INFO] checkout") || !strings.Contains(out, "served route=/x") { + t.Fatalf("pretty log line missing: %q", out) + } + if !strings.Contains(out, "\"status\": \"ok\"") || !strings.Contains(out, "\"service\": \"checkout\"") { + t.Fatalf("pretty final JSON missing: %q", out) + } +} + +func TestNonDevModeSuppressesPrettyOutput(t *testing.T) { + h := newHarness(t, Config{}) + h.captureDevOutput() + ctx := Begin(context.Background(), BeginOptions{}) + + From(ctx).Info("served") + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + if h.devBuf.Len() != 0 { + t.Fatalf("non-dev mode must not emit pretty output: %q", h.devBuf.String()) + } +} + +func TestDevModeConcurrentLoggingRemainsRaceSafe(t *testing.T) { + h := newHarness(t, Config{DevMode: true, MaxLogs: 100}) + h.captureDevOutput() + ctx := Begin(context.Background(), BeginOptions{}) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for j := 0; j < 20; j++ { + From(ctx).Info("hello", F{"n": n, "j": j}) + } + }(i) + } + wg.Wait() + + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + if !strings.Contains(h.devBuf.String(), "\"status\": \"ok\"") { + t.Fatalf("expected final pretty JSON in dev output: %q", h.devBuf.String()) + } +} diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go index 19e8dc1..6e46814 100644 --- a/pkg/waylog/v2/waylog.go +++ b/pkg/waylog/v2/waylog.go @@ -3,7 +3,8 @@ // Public surface: Init, Shutdown, Stats, Logger, From, Step, StepVoid, Fail, // NewError, Suppress, Explain. Internal-but-exported helpers for middleware // and adapter authors: Begin, Finalize, FinalizePanic, FinalizeAborted, -// FinalizeTimeout, SetField, SetHTTPStatus, TraceID, SpanID, RecordOutgoingSpan. +// FinalizeTimeout, SetField, SetHTTPStatus, SetHTTPRoute, TraceID, SpanID, +// RecordOutgoingSpan. package waylogv2 import ( @@ -12,9 +13,12 @@ import ( "fmt" "io" "os" + "strings" "sync" "sync/atomic" "time" + + transporthttp "github.com/sssmaran/WaylogCLI/pkg/transport/http" ) // F is the field bag used by Logger calls and request fields. @@ -26,12 +30,13 @@ type Config struct { Env string Version string - // Output is the writer for final wide events. Defaults to os.Stderr if - // nil and IngestURL is empty. One JSON event per line. + // Output is the local writer for final wide events when IngestURL is empty. + // Defaults to os.Stderr. One JSON event per line. Output io.Writer - // IngestURL / APIKey are accepted for API stability with future slices - // but ignored in local mode. + // IngestURL / APIKey enable HTTP delivery transport. When IngestURL is + // set, final events are submitted to /v1/events instead of being written + // to Output. Dev dual-emit remains local-only. IngestURL string APIKey string @@ -75,8 +80,13 @@ type StatsSnapshot struct { var ErrAlreadyInitialized = errors.New("waylog: SDK already initialized with active requests") type sdk struct { - cfg Config - out io.Writer + cfg Config + out io.Writer + devOut io.Writer + + devEnabled bool + delivery *transporthttp.Client + devMu sync.Mutex mu sync.Mutex active map[*request]struct{} @@ -117,15 +127,26 @@ func Init(cfg Config) error { cfg.MaxBufferBytes = defaultMaxBufferBytes } - out := cfg.Output - if out == nil { - out = os.Stderr + localOut := cfg.Output + if localOut == nil { + localOut = os.Stderr } + devEnabled := cfg.DevMode || strings.EqualFold(cfg.Env, "dev") s := &sdk{ - cfg: cfg, - out: out, - active: make(map[*request]struct{}), + cfg: cfg, + out: localOut, + devOut: os.Stderr, + devEnabled: devEnabled, + active: make(map[*request]struct{}), + } + if cfg.IngestURL != "" { + s.delivery = transporthttp.New(transporthttp.Config{ + IngestURL: cfg.IngestURL, + APIKey: cfg.APIKey, + Timeout: 5 * time.Second, + BatchMode: true, + }) } stateMu.Lock() @@ -158,6 +179,9 @@ func Shutdown(ctx context.Context) error { n := len(s.active) s.mu.Unlock() if n == 0 { + if s.delivery != nil { + s.delivery.Shutdown(remainingTimeout(ctx)) + } return nil } select { @@ -207,6 +231,17 @@ func getState() *sdk { return state } +func remainingTimeout(ctx context.Context) time.Duration { + if deadline, ok := ctx.Deadline(); ok { + d := time.Until(deadline) + if d < 0 { + return 0 + } + return d + } + return 0 +} + func resetForTest() { stateMu.Lock() state = nil From e330bf265694aef4739be4d031605318057499e7 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 05:33:33 -0400 Subject: [PATCH 5/8] feat(v2): added Go SDK delivery transport and adapter hardening Add the v2 Go HTTP delivery path and tighten the SDK/adapters around the new request lifecycle. Major changes: - add v2 HTTP transport delivery with single-event POST and NDJSON batching - add priority-aware queueing so ok/suppressed events are dropped before error, timeout, partial, and aborted events - add retry handling for transient delivery failures with bounded backoff - wire SDK IngestURL delivery so final events can go to /v1/events instead of local writer output - add drop/failure visibility through SDK stats - add shared priority status classification on eventv2.Status Hardening: - lock local SDK output writes for race-safe watchdog/late-finalize paths - reject invalid ingest URLs instead of silently dropping events - wire MaxInFlightBytes and MaxEventsPerSec pressure knobs - preserve first HTTP status on duplicate WriteHeader calls - mark Echo returned errors as Waylog failures - preserve Chi route templates during panic recovery Tests: - add transport retry/drop/pressure tests - add SDK pressure and invalid config tests - add adapter regression tests for duplicate headers, Echo errors, and Chi panic routes --- pkg/event/v2/event.go | 9 ++ pkg/transport/http/batch.go | 107 ++++++++------- pkg/transport/http/client.go | 100 ++++++++++++-- pkg/transport/http/client_test.go | 107 ++++++++++++++- pkg/transport/http/queue.go | 208 +++++++++++++++++++++++++---- pkg/waylog/chi/middleware.go | 8 +- pkg/waylog/chi/middleware_test.go | 47 ++++++- pkg/waylog/echo/middleware.go | 12 ++ pkg/waylog/echo/middleware_test.go | 27 ++++ pkg/waylog/http/middleware.go | 3 + pkg/waylog/http/middleware_test.go | 43 +++++- pkg/waylog/v2/assemble.go | 27 +++- pkg/waylog/v2/sdk_test.go | 49 +++++++ pkg/waylog/v2/waylog.go | 38 +++++- 14 files changed, 677 insertions(+), 108 deletions(-) diff --git a/pkg/event/v2/event.go b/pkg/event/v2/event.go index 9d582fe..b4e2a63 100644 --- a/pkg/event/v2/event.go +++ b/pkg/event/v2/event.go @@ -25,6 +25,15 @@ const ( StatusSuppressed Status = "suppressed" ) +func (s Status) IsPriority() bool { + switch s { + case StatusError, StatusTimeout, StatusPartial, StatusAborted: + return true + default: + return false + } +} + const ( CodeTimeout = "WAYLOG_TIMEOUT" CodeAborted = "WAYLOG_ABORTED" diff --git a/pkg/transport/http/batch.go b/pkg/transport/http/batch.go index 93fb660..2369ca0 100644 --- a/pkg/transport/http/batch.go +++ b/pkg/transport/http/batch.go @@ -5,69 +5,80 @@ import ( "encoding/json" "io" "net/http" + "strconv" + "strings" + "time" eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" ) -func (c *Client) flushBatch(batch []*eventv2.Event) { +type deliveryResult struct { + success bool + retryable bool + retryAfter time.Duration +} + +func (c *Client) flushBatch(batch []*eventv2.Event) deliveryResult { if c.url == "" || len(batch) == 0 { - return + return deliveryResult{success: true} } - for _, chunk := range splitBatches(batch, c.cfg.MaxBatch, c.cfg.MaxBatchSize) { - var body bytes.Buffer - enc := json.NewEncoder(&body) - for _, ev := range chunk { - if err := enc.Encode(ev); err != nil { - continue - } - } - req, err := http.NewRequest(http.MethodPost, c.url, bytes.NewReader(body.Bytes())) - if err != nil { - continue - } - req.Header.Set("Content-Type", "application/x-ndjson") - if c.cfg.APIKey != "" { - req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) + var body bytes.Buffer + enc := json.NewEncoder(&body) + for _, ev := range batch { + if err := enc.Encode(ev); err != nil { + c.recordDrop(1) } - resp, err := c.http.Do(req) - if err != nil { - continue - } - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() } -} + if body.Len() == 0 { + return deliveryResult{success: true} + } -func splitBatches(batch []*eventv2.Event, maxCount, maxBytes int) [][]*eventv2.Event { - if len(batch) == 0 { - return nil + req, err := http.NewRequest(http.MethodPost, c.url, bytes.NewReader(body.Bytes())) + if err != nil { + c.recordDrop(len(batch)) + return deliveryResult{} } - if maxCount <= 0 { - maxCount = len(batch) + req.Header.Set("Content-Type", "application/x-ndjson") + if c.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) } - if maxBytes <= 0 { - maxBytes = 1 << 20 + + resp, err := c.http.Do(req) + if err != nil { + c.recordFailure(len(batch)) + return deliveryResult{retryable: true} } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() - var out [][]*eventv2.Event - start := 0 - for start < len(batch) { - end := start - size := 0 - for end < len(batch) && end-start < maxCount { - next := int(estimateEventSize(batch[end])) - if end > start && size+next > maxBytes { - break - } - size += next - end++ + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return deliveryResult{success: true} + case isRetryableStatus(resp.StatusCode): + c.recordFailure(len(batch)) + return deliveryResult{ + retryable: true, + retryAfter: retryAfter(resp.Header.Get("Retry-After")), } - if end == start { - end++ + default: + c.recordDrop(len(batch)) + return deliveryResult{} + } +} + +func retryAfter(raw string) time.Duration { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0 + } + if seconds, err := strconv.Atoi(raw); err == nil { + return time.Duration(seconds) * time.Second + } + if t, err := http.ParseTime(raw); err == nil { + if d := time.Until(t); d > 0 { + return d } - out = append(out, batch[start:end]) - start = end } - return out + return 0 } diff --git a/pkg/transport/http/client.go b/pkg/transport/http/client.go index edadcca..20adc28 100644 --- a/pkg/transport/http/client.go +++ b/pkg/transport/http/client.go @@ -6,13 +6,21 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "sync" + "sync/atomic" "time" eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2" ) +const ( + defaultMaxRetries = 5 + defaultBackoffMin = 100 * time.Millisecond + defaultBackoffMax = 5 * time.Second +) + type Config struct { IngestURL string APIKey string @@ -24,6 +32,7 @@ type Config struct { BatchAgeMs int OkBudgetPct int InFlightCap int64 + MaxRetries int } type Client struct { @@ -32,9 +41,12 @@ type Client struct { http *http.Client queue *queue closed sync.Once + + dropped atomic.Int64 + failures atomic.Int64 } -func New(cfg Config) *Client { +func New(cfg Config) (*Client, error) { if cfg.Timeout <= 0 { cfg.Timeout = 5 * time.Second } @@ -53,32 +65,51 @@ func New(cfg Config) *Client { if cfg.InFlightCap <= 0 { cfg.InFlightCap = 10 << 20 } + if cfg.MaxRetries <= 0 { + cfg.MaxRetries = defaultMaxRetries + } + + ingestURL, ok := NormalizeIngestURL(cfg.IngestURL) + if !ok { + return nil, &InvalidIngestURLError{URL: cfg.IngestURL} + } c := &Client{ cfg: cfg, - url: normalizeIngestURL(cfg.IngestURL), + url: ingestURL, http: &http.Client{Timeout: cfg.Timeout}, } if cfg.BatchMode { - c.queue = newQueue(cfg, c.flushBatch) + c.queue = newQueue(cfg, c.flushBatch, c.recordDrop) go c.queue.run() } - return c + return c, nil +} + +type InvalidIngestURLError struct { + URL string } -func normalizeIngestURL(raw string) string { +func (e *InvalidIngestURLError) Error() string { + return "waylog transport: invalid ingest URL " + strconv.Quote(e.URL) +} + +func NormalizeIngestURL(raw string) (string, bool) { raw = strings.TrimRight(raw, "/") if raw == "" { - return "" + return "", true } u, err := url.Parse(raw) if err != nil || u.Scheme == "" || u.Host == "" { - return "" + return "", false } - if strings.HasSuffix(raw, "/v1/events") { - return raw + if u.Scheme != "http" && u.Scheme != "https" { + return "", false } - return raw + "/v1/events" + if strings.HasSuffix(u.Path, "/v1/events") { + return raw, true + } + return raw + "/v1/events", true } func (c *Client) Submit(ev *eventv2.Event) bool { @@ -86,7 +117,11 @@ func (c *Client) Submit(ev *eventv2.Event) bool { return false } if c.queue != nil { - return c.queue.enqueue(ev) + if c.queue.enqueue(ev) { + return true + } + c.recordDrop(1) + return false } return c.submitSingle(ev) } @@ -105,10 +140,12 @@ func (c *Client) submitSingle(ev *eventv2.Event) bool { } raw, err := json.Marshal(ev) if err != nil { + c.recordDrop(1) return false } req, err := http.NewRequest(http.MethodPost, c.url, bytes.NewReader(raw)) if err != nil { + c.recordDrop(1) return false } req.Header.Set("Content-Type", "application/json") @@ -117,9 +154,48 @@ func (c *Client) submitSingle(ev *eventv2.Event) bool { } resp, err := c.http.Do(req) if err != nil { + c.recordFailure(1) return false } _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() - return resp.StatusCode >= 200 && resp.StatusCode < 300 + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return true + } + if isRetryableStatus(resp.StatusCode) { + c.recordFailure(1) + } else { + c.recordDrop(1) + } + return false +} + +func (c *Client) Dropped() int64 { + if c == nil { + return 0 + } + return c.dropped.Load() +} + +func (c *Client) Failures() int64 { + if c == nil { + return 0 + } + return c.failures.Load() +} + +func (c *Client) recordDrop(n int) { + if n > 0 { + c.dropped.Add(int64(n)) + } +} + +func (c *Client) recordFailure(n int) { + if n > 0 { + c.failures.Add(int64(n)) + } +} + +func isRetryableStatus(code int) bool { + return code == http.StatusTooManyRequests || code >= 500 } diff --git a/pkg/transport/http/client_test.go b/pkg/transport/http/client_test.go index e4fc8df..6e74c0e 100644 --- a/pkg/transport/http/client_test.go +++ b/pkg/transport/http/client_test.go @@ -50,7 +50,10 @@ func TestClientSinglePost(t *testing.T) { })) defer srv.Close() - cli := New(Config{IngestURL: srv.URL, APIKey: "k"}) + cli, err := New(Config{IngestURL: srv.URL, APIKey: "k"}) + if err != nil { + t.Fatalf("New: %v", err) + } cli.Submit(validEvent("e1", eventv2.StatusOK)) cli.Shutdown(2 * time.Second) if got.EventID != "e1" { @@ -81,7 +84,10 @@ func TestNDJSONBatchFlush(t *testing.T) { })) defer srv.Close() - cli := New(Config{IngestURL: srv.URL, BatchMode: true, BatchAgeMs: 20}) + cli, err := New(Config{IngestURL: srv.URL, BatchMode: true, BatchAgeMs: 20}) + if err != nil { + t.Fatalf("New: %v", err) + } for i := 0; i < 300; i++ { cli.Submit(validEvent(itoa(i), eventv2.StatusOK)) } @@ -103,11 +109,12 @@ func TestQueueEvictsOKBeforePriority(t *testing.T) { BatchAgeMs: 50, OkBudgetPct: 70, InFlightCap: 600, - }, func(batch []*eventv2.Event) { + }, func(batch []*eventv2.Event) deliveryResult { for _, ev := range batch { flushed = append(flushed, ev.EventID) } - }) + return deliveryResult{success: true} + }, nil) go q.run() for i := 0; i < 6; i++ { @@ -131,6 +138,98 @@ func TestQueueEvictsOKBeforePriority(t *testing.T) { } } +func TestQueuePressureEvictionsIncrementDrops(t *testing.T) { + var drops atomic.Int64 + q := newQueue(Config{ + MaxBatch: 256, + MaxBatchSize: 1 << 20, + BatchAgeMs: 50, + OkBudgetPct: 70, + InFlightCap: 600, + }, func(batch []*eventv2.Event) deliveryResult { + return deliveryResult{success: true} + }, func(n int) { + drops.Add(int64(n)) + }) + + for i := 0; i < 4; i++ { + q.enqueue(validEvent("ok-"+itoa(i), eventv2.StatusOK)) + } + + if drops.Load() == 0 { + t.Fatal("expected queue pressure evictions to increment drops") + } +} + +func TestNewRejectsInvalidIngestURL(t *testing.T) { + if _, err := New(Config{IngestURL: "localhost:8080"}); err == nil { + t.Fatal("expected invalid ingest URL error") + } +} + +func TestBatchRetriesTransientFailure(t *testing.T) { + var calls atomic.Int64 + var totalEvents atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + sc := bufio.NewScanner(r.Body) + for sc.Scan() { + totalEvents.Add(1) + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cli, err := New(Config{ + IngestURL: srv.URL, + BatchMode: true, + BatchAgeMs: 1, + MaxRetries: 2, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + cli.Submit(validEvent("retry-me", eventv2.StatusError)) + deadline := time.After(2 * time.Second) + for calls.Load() < 2 { + select { + case <-deadline: + t.Fatalf("expected retry after transient failure, got %d calls", calls.Load()) + default: + time.Sleep(10 * time.Millisecond) + } + } + cli.Shutdown(2 * time.Second) + + if totalEvents.Load() != 1 { + t.Fatalf("events delivered=%d want 1", totalEvents.Load()) + } + if cli.Failures() == 0 { + t.Fatal("transient failure should increment failure counter") + } +} + +func TestBatchDropsPermanentFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer srv.Close() + + cli, err := New(Config{IngestURL: srv.URL, BatchMode: true, BatchAgeMs: 1}) + if err != nil { + t.Fatalf("New: %v", err) + } + cli.Submit(validEvent("bad", eventv2.StatusError)) + cli.Shutdown(2 * time.Second) + + if cli.Dropped() != 1 { + t.Fatalf("Dropped=%d want 1", cli.Dropped()) + } +} + func itoa(i int) string { if i == 0 { return "0" diff --git a/pkg/transport/http/queue.go b/pkg/transport/http/queue.go index c7bbd8c..0352cf4 100644 --- a/pkg/transport/http/queue.go +++ b/pkg/transport/http/queue.go @@ -1,7 +1,6 @@ package transporthttp import ( - "bytes" "encoding/json" "sync" "time" @@ -10,13 +9,15 @@ import ( ) type queuedEvent struct { - ev *eventv2.Event - size int64 + ev *eventv2.Event + size int64 + attempts int } type queue struct { cfg Config - flushFn func(batch []*eventv2.Event) + flushFn func(batch []*eventv2.Event) deliveryResult + dropFn func(n int) mu sync.Mutex okQ []queuedEvent @@ -29,10 +30,23 @@ type queue struct { drained chan struct{} } -func newQueue(cfg Config, flushFn func(batch []*eventv2.Event)) *queue { +type batchClass uint8 + +const ( + classOK batchClass = iota + classPriority +) + +type queuedBatch struct { + class batchClass + events []queuedEvent +} + +func newQueue(cfg Config, flushFn func(batch []*eventv2.Event) deliveryResult, dropFn func(n int)) *queue { return &queue{ cfg: cfg, flushFn: flushFn, + dropFn: dropFn, notify: make(chan struct{}, 1), stop: make(chan struct{}), drained: make(chan struct{}), @@ -52,11 +66,10 @@ func (q *queue) enqueue(ev *eventv2.Event) bool { } accepted := false - switch ev.Status { - case eventv2.StatusOK, eventv2.StatusSuppressed: - accepted = q.enqueueOKLocked(ev, size) - default: + if ev.Status.IsPriority() { accepted = q.enqueuePriorityLocked(ev, size) + } else { + accepted = q.enqueueOKLocked(ev, size) } if accepted { q.signal() @@ -70,8 +83,7 @@ func (q *queue) enqueueOKLocked(ev *eventv2.Event, size int64) bool { return false } for q.okBytes+size > okCap && len(q.okQ) > 0 { - q.okBytes -= q.okQ[0].size - q.okQ = q.okQ[1:] + q.dropOKLocked() } if q.okBytes+size > okCap { return false @@ -87,12 +99,10 @@ func (q *queue) enqueuePriorityLocked(ev *eventv2.Event, size int64) bool { return false } for q.totalBytesLocked()+size > totalCap && len(q.okQ) > 0 { - q.okBytes -= q.okQ[0].size - q.okQ = q.okQ[1:] + q.dropOKLocked() } for q.totalBytesLocked()+size > totalCap && len(q.prioQ) > 0 { - q.prioBytes -= q.prioQ[0].size - q.prioQ = q.prioQ[1:] + q.dropPriorityLocked() } if q.totalBytesLocked()+size > totalCap { return false @@ -115,46 +125,46 @@ func (q *queue) run() { select { case <-q.notify: for _, batch := range q.takeReadyBatches(false) { - q.flushFn(batch) + q.deliver(batch) } case <-ticker.C: for _, batch := range q.takeReadyBatches(true) { - q.flushFn(batch) + q.deliver(batch) } case <-q.stop: for _, batch := range q.takeReadyBatches(true) { - q.flushFn(batch) + q.deliver(batch) } return } } } -func (q *queue) takeReadyBatches(force bool) [][]*eventv2.Event { +func (q *queue) takeReadyBatches(force bool) []queuedBatch { q.mu.Lock() defer q.mu.Unlock() - var out [][]*eventv2.Event + var out []queuedBatch for { batch := q.takeOneLocked(force) - if len(batch) == 0 { + if len(batch.events) == 0 { return out } out = append(out, batch) } } -func (q *queue) takeOneLocked(force bool) []*eventv2.Event { +func (q *queue) takeOneLocked(force bool) queuedBatch { if batch := q.takeFromLocked(&q.prioQ, &q.prioBytes, force); len(batch) > 0 { - return batch + return queuedBatch{class: classPriority, events: batch} } if batch := q.takeFromLocked(&q.okQ, &q.okBytes, force); len(batch) > 0 { - return batch + return queuedBatch{class: classOK, events: batch} } - return nil + return queuedBatch{} } -func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool) []*eventv2.Event { +func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool) []queuedEvent { if len(*src) == 0 { return nil } @@ -162,7 +172,7 @@ func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool return nil } - var batch []*eventv2.Event + var batch []queuedEvent var batchBytes int64 n := 0 for n < len(*src) && n < q.cfg.MaxBatch { @@ -170,7 +180,7 @@ func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool if n > 0 && batchBytes+next.size > int64(q.cfg.MaxBatchSize) { break } - batch = append(batch, next.ev) + batch = append(batch, next) batchBytes += next.size n++ } @@ -179,6 +189,146 @@ func (q *queue) takeFromLocked(src *[]queuedEvent, totalBytes *int64, force bool return batch } +func (q *queue) deliver(batch queuedBatch) { + events := make([]*eventv2.Event, 0, len(batch.events)) + for _, item := range batch.events { + events = append(events, item.ev) + } + + result := q.flushFn(events) + if result.success { + return + } + if !result.retryable { + return + } + + retry := queuedBatch{class: batch.class} + for _, ev := range batch.events { + ev.attempts++ + if ev.attempts > q.cfg.MaxRetries { + if q.dropFn != nil { + q.dropFn(1) + } + continue + } + retry.events = append(retry.events, ev) + } + if len(retry.events) == 0 { + return + } + if q.isClosing() { + if q.dropFn != nil { + q.dropFn(len(retry.events)) + } + return + } + + // A failing endpoint stalls the single delivery loop for the backoff + // window; switch to time.AfterFunc + signal if retry storms become a + // throughput problem. + if result.retryAfter > 0 { + time.Sleep(result.retryAfter) + } else { + time.Sleep(q.backoff(retry.events)) + } + if q.requeueFront(retry) { + q.signal() + } else if q.dropFn != nil { + q.dropFn(len(retry.events)) + } +} + +func (q *queue) backoff(events []queuedEvent) time.Duration { + maxAttempt := 1 + for _, ev := range events { + if ev.attempts > maxAttempt { + maxAttempt = ev.attempts + } + } + d := defaultBackoffMin << (maxAttempt - 1) + if d > defaultBackoffMax { + return defaultBackoffMax + } + return d +} + +func (q *queue) requeueFront(batch queuedBatch) bool { + q.mu.Lock() + defer q.mu.Unlock() + if q.closing { + return false + } + + var bytes int64 + for _, ev := range batch.events { + bytes += ev.size + } + switch batch.class { + case classPriority: + totalCap := q.cfg.InFlightCap + for q.totalBytesLocked()+bytes > totalCap && len(q.okQ) > 0 { + q.dropOKLocked() + } + for q.totalBytesLocked()+bytes > totalCap && len(q.prioQ) > 0 { + q.dropPriorityTailLocked() + } + if q.totalBytesLocked()+bytes > totalCap { + return false + } + q.prioQ = append(append([]queuedEvent(nil), batch.events...), q.prioQ...) + q.prioBytes += bytes + default: + okCap := q.cfg.InFlightCap * int64(q.cfg.OkBudgetPct) / 100 + for q.okBytes+bytes > okCap && len(q.okQ) > 0 { + q.dropOKLocked() + } + if q.okBytes+bytes > okCap { + return false + } + q.okQ = append(append([]queuedEvent(nil), batch.events...), q.okQ...) + q.okBytes += bytes + } + return true +} + +func (q *queue) dropOKLocked() { + q.dropHeadLocked(&q.okQ, &q.okBytes) +} + +func (q *queue) dropPriorityLocked() { + q.dropHeadLocked(&q.prioQ, &q.prioBytes) +} + +func (q *queue) dropHeadLocked(src *[]queuedEvent, totalBytes *int64) { + if len(*src) == 0 { + return + } + *totalBytes -= (*src)[0].size + *src = (*src)[1:] + if q.dropFn != nil { + q.dropFn(1) + } +} + +func (q *queue) dropPriorityTailLocked() { + if len(q.prioQ) == 0 { + return + } + last := len(q.prioQ) - 1 + q.prioBytes -= q.prioQ[last].size + q.prioQ = q.prioQ[:last] + if q.dropFn != nil { + q.dropFn(1) + } +} + +func (q *queue) isClosing() bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.closing +} + func (q *queue) shutdown(timeout time.Duration) { q.mu.Lock() if q.closing { @@ -212,5 +362,5 @@ func estimateEventSize(ev *eventv2.Event) int64 { if err != nil { return 0 } - return int64(len(bytes.TrimSpace(raw)) + 1) + return int64(len(raw) + 1) } diff --git a/pkg/waylog/chi/middleware.go b/pkg/waylog/chi/middleware.go index b986cde..423b960 100644 --- a/pkg/waylog/chi/middleware.go +++ b/pkg/waylog/chi/middleware.go @@ -18,10 +18,12 @@ func Middleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route := "" wayloghttp.ServeHTTP(w, r, route, func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rc := chi.RouteContext(r.Context()); rc != nil { + waylogv2.SetHTTPRoute(r.Context(), rc.RoutePattern()) + } + }() next.ServeHTTP(w, r) - if rc := chi.RouteContext(r.Context()); rc != nil { - waylogv2.SetHTTPRoute(r.Context(), rc.RoutePattern()) - } }) }) } diff --git a/pkg/waylog/chi/middleware_test.go b/pkg/waylog/chi/middleware_test.go index 0a09337..e320d4f 100644 --- a/pkg/waylog/chi/middleware_test.go +++ b/pkg/waylog/chi/middleware_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "sync" "testing" "time" @@ -17,13 +18,30 @@ import ( const schemaPath = "../../../docs/schema/v2.0.json" -func newHarness(t *testing.T, cfg waylogv2.Config) *bytes.Buffer { +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + return append([]byte(nil), b.buf.Bytes()...) +} + +func newHarness(t *testing.T, cfg waylogv2.Config) *lockedBuffer { t.Helper() deadline, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() _ = waylogv2.Shutdown(deadline) - buf := &bytes.Buffer{} + buf := &lockedBuffer{} if cfg.Service == "" { cfg.Service = "checkout" } @@ -37,7 +55,7 @@ func newHarness(t *testing.T, cfg waylogv2.Config) *bytes.Buffer { return buf } -func lastEvent(t *testing.T, buf *bytes.Buffer) *eventv2.Event { +func lastEvent(t *testing.T, buf *lockedBuffer) *eventv2.Event { t.Helper() var ev eventv2.Event if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &ev); err != nil { @@ -91,3 +109,26 @@ func TestMiddlewareTimeoutUsesSharedLifecycle(t *testing.T) { t.Fatalf("timeout anchor wrong: %+v", ev.Anchor) } } + +func TestMiddlewarePanicKeepsRouteTemplate(t *testing.T) { + buf := newHarness(t, waylogv2.Config{}) + + r := chi.NewRouter() + r.Use(Middleware) + r.Get("/panic/{id}", func(w http.ResponseWriter, r *http.Request) { + panic("boom") + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/panic/42", nil) + r.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + httpFields, _ := ev.Fields["http"].(map[string]any) + if got := httpFields["route"]; got != "/panic/{id}" { + t.Fatalf("route=%v want /panic/{id}", got) + } +} diff --git a/pkg/waylog/echo/middleware.go b/pkg/waylog/echo/middleware.go index adf289a..7819871 100644 --- a/pkg/waylog/echo/middleware.go +++ b/pkg/waylog/echo/middleware.go @@ -1,11 +1,14 @@ package waylogecho import ( + "errors" + "fmt" "net/http" "github.com/labstack/echo/v4" wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" ) // Middleware applies the shared Waylog v2 HTTP lifecycle to Echo handlers @@ -17,6 +20,7 @@ func Middleware() echo.MiddlewareFunc { c.SetRequest(r) c.Response().Writer = w if err := next(c); err != nil { + waylogv2.Fail(r.Context(), waylogv2.NewError(echoErrorCode(err), waylogv2.WithReason(err.Error()))) c.Error(err) } }) @@ -24,3 +28,11 @@ func Middleware() echo.MiddlewareFunc { } } } + +func echoErrorCode(err error) string { + var he *echo.HTTPError + if errors.As(err, &he) { + return fmt.Sprintf("HTTP_%d", he.Code) + } + return "ERR" +} diff --git a/pkg/waylog/echo/middleware_test.go b/pkg/waylog/echo/middleware_test.go index ed9fcee..85b93e7 100644 --- a/pkg/waylog/echo/middleware_test.go +++ b/pkg/waylog/echo/middleware_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "testing" @@ -88,3 +89,29 @@ func TestMiddlewareKeepsSuppressedStatus(t *testing.T) { t.Fatalf("status=%s want suppressed", ev.Status) } } + +func TestMiddlewareReturnedErrorBecomesWaylogFailure(t *testing.T) { + buf := newHarness(t, waylogv2.Config{}) + + e := echo.New() + e.Use(Middleware()) + e.GET("/boom/:id", func(c echo.Context) error { + return errors.New("boom") + }) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/boom/42", nil) + e.ServeHTTP(rr, req) + + ev := lastEvent(t, buf) + if ev.Status != eventv2.StatusError { + t.Fatalf("status=%s want error", ev.Status) + } + if ev.Anchor == nil || ev.Anchor.Step != "request" || ev.Anchor.ErrorCode != "ERR" { + t.Fatalf("anchor wrong: %+v", ev.Anchor) + } + httpFields, _ := ev.Fields["http"].(map[string]any) + if got, _ := httpFields["status"].(float64); got != 500 { + t.Fatalf("fields.http.status=%v want 500", httpFields["status"]) + } +} diff --git a/pkg/waylog/http/middleware.go b/pkg/waylog/http/middleware.go index 8d3cd17..7b79c5b 100644 --- a/pkg/waylog/http/middleware.go +++ b/pkg/waylog/http/middleware.go @@ -159,6 +159,9 @@ func (w *statusWriter) Header() http.Header { } func (w *statusWriter) WriteHeader(statusCode int) { + if w.wroteHeader { + return + } w.status = statusCode w.wroteHeader = true waylogv2.SetHTTPStatus(w.ctx, statusCode) diff --git a/pkg/waylog/http/middleware_test.go b/pkg/waylog/http/middleware_test.go index 5c887d5..217cea3 100644 --- a/pkg/waylog/http/middleware_test.go +++ b/pkg/waylog/http/middleware_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -21,7 +22,7 @@ const schemaPath = "../../../docs/schema/v2.0.json" type harness struct { t *testing.T - buf *bytes.Buffer + buf *lockedBuffer } func newHarness(t *testing.T, cfg waylogv2.Config) *harness { @@ -30,7 +31,7 @@ func newHarness(t *testing.T, cfg waylogv2.Config) *harness { defer cancel() _ = waylogv2.Shutdown(deadline) - buf := &bytes.Buffer{} + buf := &lockedBuffer{} if cfg.Service == "" { cfg.Service = "checkout" } @@ -58,6 +59,23 @@ func (h *harness) lastEvent() *eventv2.Event { return &ev } +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + return append([]byte(nil), b.buf.Bytes()...) +} + func (h *harness) validateLast() *eventv2.Event { h.t.Helper() ev := h.lastEvent() @@ -141,6 +159,27 @@ func TestHTTPMiddlewareWriteWithoutHeaderRecords200(t *testing.T) { } } +func TestHTTPMiddlewareKeepsFirstWriteHeaderStatus(t *testing.T) { + h := newHarness(t, waylogv2.Config{}) + handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusInternalServerError) + })) + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/double-header", nil) + handler.ServeHTTP(rr, req) + + ev := h.validateLast() + fields := httpFields(t, ev) + if got, _ := fields["status"].(float64); got != 204 { + t.Fatalf("fields.http.status=%v want 204", fields["status"]) + } + if rr.Code != http.StatusNoContent { + t.Fatalf("response code=%d want 204", rr.Code) + } +} + func TestHTTPMiddlewarePreservesTraceparent(t *testing.T) { h := newHarness(t, waylogv2.Config{}) handler := HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index f1ff947..33183da 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -125,8 +125,6 @@ func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { } if len(r.fields) > 0 { - // Copy only when a Redactor will mutate the result; otherwise hand - // the sealed request's map directly to the event. if r.sdk.cfg.Redactor != nil { ev.Fields = make(map[string]any, len(r.fields)) maps.Copy(ev.Fields, r.fields) @@ -214,11 +212,36 @@ func emit(w io.Writer, ev *eventv2.Event) error { } func deliver(s *sdk, ev *eventv2.Event) (bool, error) { + if !s.allowEvent(ev) { + s.eventsDropped.Add(1) + return false, nil + } if s.delivery != nil { return s.delivery.Submit(ev), nil } + s.emitMu.Lock() + defer s.emitMu.Unlock() if err := emit(s.out, ev); err != nil { return false, err } return true, nil } + +func (s *sdk) allowEvent(ev *eventv2.Event) bool { + if s.cfg.MaxEventsPerSec <= 0 || ev == nil || ev.Status.IsPriority() { + return true + } + + now := time.Now().Unix() + s.rateMu.Lock() + defer s.rateMu.Unlock() + if s.rateSecond != now { + s.rateSecond = now + s.rateCount = 0 + } + if s.rateCount >= s.cfg.MaxEventsPerSec { + return false + } + s.rateCount++ + return true +} diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index fd07519..b94dc66 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -778,6 +778,55 @@ func TestIngestURLRoutesFinalEventToTransport(t *testing.T) { } } +func TestInvalidIngestURLRejectsInit(t *testing.T) { + resetForTest() + if err := Init(Config{Service: "checkout", Env: "test", IngestURL: "localhost:8080"}); err == nil { + t.Fatal("expected invalid IngestURL error") + } + if getState() != nil { + t.Fatal("invalid Init must not install SDK state") + } +} + +func TestMaxInFlightBytesDropsOversizedTransportEvent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("oversized event should be rejected before transport POST") + })) + defer srv.Close() + + newHarness(t, Config{IngestURL: srv.URL, MaxInFlightBytes: 32}) + ctx := Begin(context.Background(), BeginOptions{}) + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + + if got := Stats().EventsDropped; got != 1 { + t.Fatalf("EventsDropped=%d want 1", got) + } +} + +func TestMaxEventsPerSecDropsOKBeforePriority(t *testing.T) { + h := newHarness(t, Config{MaxEventsPerSec: 1}) + + ok1 := Begin(context.Background(), BeginOptions{}) + _, _ = Finalize(ok1) + + ok2 := Begin(context.Background(), BeginOptions{}) + _, _ = Finalize(ok2) + + boom := Begin(context.Background(), BeginOptions{}) + Fail(boom, NewError("BOOM")) + _, _ = Finalize(boom) + + lines := bytes.Split(bytes.TrimSpace(h.buf.Bytes()), []byte{'\n'}) + if len(lines) != 2 { + t.Fatalf("emitted lines=%d want 2; output=%s", len(lines), h.buf.String()) + } + if got := Stats().EventsDropped; got != 1 { + t.Fatalf("EventsDropped=%d want 1", got) + } +} + func TestDevModeEmitsPrettyLogsAndFinalEvent(t *testing.T) { h := newHarness(t, Config{DevMode: true}) h.captureDevOutput() diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go index 6e46814..183391a 100644 --- a/pkg/waylog/v2/waylog.go +++ b/pkg/waylog/v2/waylog.go @@ -73,6 +73,8 @@ type StatsSnapshot struct { ReservedCodeRejections int64 // NewError/Fail/Step returns with WAYLOG_* codes SuppressedThenFailed int64 LateCompletionAfterEmit int64 + EventsDropped int64 + DeliveryFailures int64 } // ErrAlreadyInitialized is returned by Init when a prior SDK is still alive @@ -86,7 +88,11 @@ type sdk struct { devEnabled bool delivery *transporthttp.Client + emitMu sync.Mutex devMu sync.Mutex + rateMu sync.Mutex + rateSecond int64 + rateCount int mu sync.Mutex active map[*request]struct{} @@ -100,6 +106,7 @@ type sdk struct { reservedRejected atomic.Int64 suppressFailed atomic.Int64 lateAfterEmit atomic.Int64 + eventsDropped atomic.Int64 } var ( @@ -141,12 +148,17 @@ func Init(cfg Config) error { active: make(map[*request]struct{}), } if cfg.IngestURL != "" { - s.delivery = transporthttp.New(transporthttp.Config{ - IngestURL: cfg.IngestURL, - APIKey: cfg.APIKey, - Timeout: 5 * time.Second, - BatchMode: true, + delivery, err := transporthttp.New(transporthttp.Config{ + IngestURL: cfg.IngestURL, + APIKey: cfg.APIKey, + Timeout: 5 * time.Second, + BatchMode: true, + InFlightCap: cfg.MaxInFlightBytes, }) + if err != nil { + return err + } + s.delivery = delivery } stateMu.Lock() @@ -212,6 +224,8 @@ func Stats() StatsSnapshot { ReservedCodeRejections: s.reservedRejected.Load(), SuppressedThenFailed: s.suppressFailed.Load(), LateCompletionAfterEmit: s.lateAfterEmit.Load(), + EventsDropped: s.eventsDropped.Load() + deliveryDropped(s), + DeliveryFailures: deliveryFailures(s), } } @@ -242,6 +256,20 @@ func remainingTimeout(ctx context.Context) time.Duration { return 0 } +func deliveryDropped(s *sdk) int64 { + if s == nil || s.delivery == nil { + return 0 + } + return s.delivery.Dropped() +} + +func deliveryFailures(s *sdk) int64 { + if s == nil || s.delivery == nil { + return 0 + } + return s.delivery.Failures() +} + func resetForTest() { stateMu.Lock() state = nil From 4d2b502eb2ba1a62a8398f26598d4a06559d79b8 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 19:27:40 -0400 Subject: [PATCH 6/8] test(v2): added Go SDK parity runner and bench gate Added Phase 1a wrap-up verification for the Go v2 SDK. Major changes: - add fixture-driven parity tests for all six v2 golden scenarios - fix SDK lifecycle semantics required by parity: - panic reason matches the fixture contract - suppressed events preserve top-level fields - timeout and aborted anchors do not add errors[] entries - timeout/panic/cancel finalization snapshots active steps before sealing - add Go benchmarks for middleware, step, logger, and 20-step/50-log assembly paths - add scripts/bench-gate.sh as a regression gate for Go v2 SDK perf - add optional make bench-gate target - document Go allocation budget debt as post-v2 stabilization work Verification: - fixture schema validation passes - v2 parity tests pass 6/6 - Go v2 bench gate passes - make ci passes --- Makefile | 5 +- pkg/waylog/v2/assemble.go | 54 +++- pkg/waylog/v2/bench/middleware_bench_test.go | 44 +++ pkg/waylog/v2/bench/step_bench_test.go | 115 ++++++++ pkg/waylog/v2/parity/parity_test.go | 272 +++++++++++++++++++ pkg/waylog/v2/parity/runner.go | 251 +++++++++++++++++ pkg/waylog/v2/request.go | 5 +- pkg/waylog/v2/sdk_test.go | 3 +- pkg/waylog/v2/step.go | 11 +- scripts/bench-gate.sh | 122 +++++++++ 10 files changed, 867 insertions(+), 15 deletions(-) create mode 100644 pkg/waylog/v2/bench/middleware_bench_test.go create mode 100644 pkg/waylog/v2/bench/step_bench_test.go create mode 100644 pkg/waylog/v2/parity/parity_test.go create mode 100644 pkg/waylog/v2/parity/runner.go create mode 100755 scripts/bench-gate.sh diff --git a/Makefile b/Makefile index 8932199..1db8e11 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/sh -.PHONY: help build build-examples ingest ingest-mcp waylog waylog-live checkout test test-race test-sdk lint ci fmt vet vet-sdk clean kafka-up kafka-down demo demo-stop micro-demo micro-demo-stop docker-build docker-up docker-down docker-reset docker-dev docker-prod ts-install ts-build ts-test +.PHONY: help build build-examples ingest ingest-mcp waylog waylog-live checkout test test-race test-sdk lint ci fmt vet vet-sdk clean kafka-up kafka-down demo demo-stop micro-demo micro-demo-stop docker-build docker-up docker-down docker-reset docker-dev docker-prod ts-install ts-build ts-test bench-gate help: @echo "Targets:" @@ -99,6 +99,9 @@ check-doc-links: check-rollup-contract: @bash scripts/check-rollup-contract.sh +bench-gate: ## Enforce v2 SDK §4.4.1 perf budgets (optional; not in `ci` yet) + @bash scripts/bench-gate.sh + clean: rm -f ingest checkout waylog bridge api-gateway checkout-demo db-demo payment-demo waylog-live diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index 33183da..b9fac2d 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -91,18 +91,56 @@ func (r *request) applyLifecycleLocked(lifecycle lifecycleKind) { return } + now := time.Now() switch lifecycle { case lifecyclePanic: - r.markLifecycleLocked(eventv2.StatusError, eventv2.CodePanic, "panic recovered") + // Panic is an app-observable failure, so it synthesizes both an + // anchor and an entry in errors[] (§3.4: errors[] is the deduped + // list of errors seen across the request). + r.markLifecycleLocked(eventv2.StatusError, eventv2.CodePanic) + r.recordErrorLocked(eventv2.CodePanic, "runtime panic recovered") + r.flushActiveStepsLocked(now) case lifecycleTimeout: - r.markLifecycleLocked(eventv2.StatusTimeout, eventv2.CodeTimeout, "") + // Watchdog expiry is a runtime decision, not an app error; the + // lifecycle code lives only in anchor, not errors[]. + r.markLifecycleLocked(eventv2.StatusTimeout, eventv2.CodeTimeout) + r.flushActiveStepsLocked(now) case lifecycleAborted: + // Cooperative cancel from the client; same errors[] rule as timeout. + // Preserve any explicit Fail anchor that was recorded before cancel. if r.anchorStep == "" { - r.markLifecycleLocked(eventv2.StatusAborted, eventv2.CodeAborted, "") + r.markLifecycleLocked(eventv2.StatusAborted, eventv2.CodeAborted) } + r.flushActiveStepsLocked(now) } } +// flushActiveStepsLocked snapshots any steps still on the active stack as +// closed steps (status=ok) so a watchdog/panic/cancel that fires while a +// step is in flight still records that the step ran. Spec §3.5: when +// lifecycle precedence wins the seal, the active step at fire time is +// captured both as the anchor and as a step entry. +func (r *request) flushActiveStepsLocked(now time.Time) { + if len(r.stepStack) == 0 { + return + } + for _, active := range r.stepStack { + durMS := int64(now.Sub(active.startedAt) / 1e6) + if durMS < 0 { + durMS = 0 + } + r.addStepLocked(stepBuf{ + name: active.name, + spanID: active.spanID, + startMS: active.startMS, + durationMS: durMS, + status: stepStatusOK, + downstream: active.downstream, + }) + } + r.stepStack = nil +} + func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { ev := &eventv2.Event{ SchemaVersion: eventv2.SchemaVersion2, @@ -120,10 +158,6 @@ func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { Status: r.snapshotStatusLocked(), } - if ev.Status == eventv2.StatusSuppressed { - return ev - } - if len(r.fields) > 0 { if r.sdk.cfg.Redactor != nil { ev.Fields = make(map[string]any, len(r.fields)) @@ -134,6 +168,12 @@ func (r *request) assembleLocked(tsEnd time.Time) *eventv2.Event { } } + // Suppressed events keep top-level identity, timing, and fields (§4.2.2) + // but drop anchor/steps/logs/errors so they remain header-only triage. + if ev.Status == eventv2.StatusSuppressed { + return ev + } + if r.anchorStep != "" && r.anchorCode != "" { ev.Anchor = &eventv2.Anchor{Step: r.anchorStep, ErrorCode: r.anchorCode} } diff --git a/pkg/waylog/v2/bench/middleware_bench_test.go b/pkg/waylog/v2/bench/middleware_bench_test.go new file mode 100644 index 0000000..33493df --- /dev/null +++ b/pkg/waylog/v2/bench/middleware_bench_test.go @@ -0,0 +1,44 @@ +package bench + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +// BenchmarkMiddlewareNoOp measures the per-request overhead of the +// net/http middleware wrapping a handler that returns 200 with no body. +// §4.4.1 budget: ≤ 33000 ns/op, ≤ 20 allocs/op. +func BenchmarkMiddlewareNoOp(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := waylogv2.Shutdown(ctx); err != nil { + b.Fatalf("pre-bench drain: %v", err) + } + if err := waylogv2.Init(waylogv2.Config{ + Service: "bench", + Env: "test", + Output: io.Discard, + }); err != nil { + b.Fatalf("Init: %v", err) + } + + handler := wayloghttp.HTTP(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + req := httptest.NewRequest(http.MethodGet, "/bench", nil) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + } +} diff --git a/pkg/waylog/v2/bench/step_bench_test.go b/pkg/waylog/v2/bench/step_bench_test.go new file mode 100644 index 0000000..29b2c4e --- /dev/null +++ b/pkg/waylog/v2/bench/step_bench_test.go @@ -0,0 +1,115 @@ +package bench + +import ( + "context" + "fmt" + "io" + "testing" + "time" + + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +const ( + benchRequestChunk = 4096 + benchMaxBufferBytes = 1 << 30 +) + +func benchInit(b *testing.B) { + b.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := waylogv2.Shutdown(ctx); err != nil { + b.Fatalf("pre-bench drain: %v", err) + } + cfg := waylogv2.Config{ + Service: "bench", + Env: "test", + Output: io.Discard, + MaxSteps: benchRequestChunk, + MaxLogs: benchRequestChunk, + MaxBufferBytes: benchMaxBufferBytes, + } + if err := waylogv2.Init(cfg); err != nil { + b.Fatalf("Init: %v", err) + } +} + +// BenchmarkStepEmptyBody measures the per-call overhead of opening and +// closing a Step with an empty body. §4.4.1 budget: ≤ 5500 ns/op, +// ≤ 4 allocs/op. +func BenchmarkStepEmptyBody(b *testing.B) { + benchInit(b) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + noop := func(ctx context.Context) error { return nil } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i > 0 && i%benchRequestChunk == 0 { + b.StopTimer() + _, _ = waylogv2.Finalize(ctx) + ctx = waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + b.StartTimer() + } + _ = waylogv2.StepVoid(ctx, "bench.step", noop) + } + b.StopTimer() + _, _ = waylogv2.Finalize(ctx) +} + +// BenchmarkLoggerInfo measures the per-call overhead of From(ctx).Info +// with a single field. §4.4.1 budget: ≤ 3300 ns/op, ≤ 3 allocs/op. +func BenchmarkLoggerInfo(b *testing.B) { + benchInit(b) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + logger := waylogv2.From(ctx) + fields := waylogv2.F{"k": "v"} + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i > 0 && i%benchRequestChunk == 0 { + b.StopTimer() + _, _ = waylogv2.Finalize(ctx) + ctx = waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + logger = waylogv2.From(ctx) + b.StartTimer() + } + logger.Info("hello", fields) + } + b.StopTimer() + _, _ = waylogv2.Finalize(ctx) +} + +// BenchmarkAssemble20Steps50Logs measures the cost of buffering a +// representative request — 20 closed steps and 50 logs — and emitting +// the final wide event. §4.4.1 budget: ≤ 110000 ns/op, ≤ 200 allocs/op. +func BenchmarkAssemble20Steps50Logs(b *testing.B) { + benchInit(b) + + noop := func(ctx context.Context) error { return nil } + logFields := waylogv2.F{"k": "v"} + logMessages := make([]string, 50) + for i := range logMessages { + logMessages[i] = fmt.Sprintf("event_%d", i) + } + stepNames := make([]string, 20) + for i := range stepNames { + stepNames[i] = fmt.Sprintf("step_%d", i) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{}) + logger := waylogv2.From(ctx) + for _, msg := range logMessages { + logger.Info(msg, logFields) + } + for _, name := range stepNames { + _ = waylogv2.StepVoid(ctx, name, noop) + } + _, _ = waylogv2.Finalize(ctx) + } +} diff --git a/pkg/waylog/v2/parity/parity_test.go b/pkg/waylog/v2/parity/parity_test.go new file mode 100644 index 0000000..451734c --- /dev/null +++ b/pkg/waylog/v2/parity/parity_test.go @@ -0,0 +1,272 @@ +package parity + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + waylogv2 "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +// scenarioRunner carries per-scenario plumbing: a fresh SDK Init, the +// emit buffer the SDK writes to, and the *eventv2.Event captured from +// the relevant Finalize* call. +type scenarioRunner struct { + t *testing.T + buf *bytes.Buffer +} + +func newScenario(t *testing.T) *scenarioRunner { + t.Helper() + if err := drainSDK(t); err != nil { + t.Fatalf("pre-test drain: %v", err) + } + buf := &bytes.Buffer{} + cfg := waylogv2.Config{ + Service: "checkout", + Env: "test", + Output: buf, + } + if err := waylogv2.Init(cfg); err != nil { + t.Fatalf("Init: %v", err) + } + return &scenarioRunner{t: t, buf: buf} +} + +// drainSDK uses the public API to wait for in-flight requests to finish +// from any prior scenario. It is the public-API-only equivalent of +// resetForTest(). +func drainSDK(t *testing.T) error { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + return waylogv2.Shutdown(ctx) +} + +// emitted parses the most recent JSON line written to the SDK output. +// All scenarios produce exactly one final event. +func (r *scenarioRunner) emitted() []byte { + r.t.Helper() + out := r.buf.Bytes() + if len(out) == 0 { + r.t.Fatal("no event emitted") + } + lines := bytes.Split(bytes.TrimRight(out, "\n"), []byte{'\n'}) + return lines[len(lines)-1] +} + +// assertParity compares the SDK's emitted JSON against the named fixture +// after masking non-deterministic fields. On mismatch it prints both +// blobs side by side so the diff is human-readable in test output. +func (r *scenarioRunner) assertParity(name string) { + r.t.Helper() + got := r.emitted() + want, err := LoadFixture(name) + if err != nil { + r.t.Fatalf("load fixture %s: %v", name, err) + } + eq, err := Equal(got, want) + if err != nil { + r.t.Fatalf("compare: %v", err) + } + if !eq { + gotPretty := jsonPretty(r.t, got) + wantPretty := jsonPretty(r.t, want) + gotMasked := jsonPretty(r.t, mustMask(r.t, got)) + wantMasked := jsonPretty(r.t, mustMask(r.t, want)) + r.t.Fatalf("parity mismatch for %s\n--- emitted (raw) ---\n%s\n--- fixture (raw) ---\n%s\n--- emitted (masked) ---\n%s\n--- fixture (masked) ---\n%s", + name, gotPretty, wantPretty, gotMasked, wantMasked) + } +} + +func mustMask(t *testing.T, raw []byte) []byte { + t.Helper() + out, err := MaskNonDeterministic(raw) + if err != nil { + t.Fatalf("mask: %v", err) + } + return out +} + +func jsonPretty(t *testing.T, raw []byte) string { + t.Helper() + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return string(raw) + } + out, err := json.MarshalIndent(v, "", " ") + if err != nil { + return string(raw) + } + return string(out) +} + +// --- Scenario 1: ok-simple -------------------------------------------------- + +func TestParity_OKSimple(t *testing.T) { + r := newScenario(t) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: strings.Repeat("1", 32), + SpanID: strings.Repeat("1", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "POST", + "route": "/checkout", + "status": 200, + }) + if err := waylogv2.StepVoid(ctx, "db.load_cart", func(ctx context.Context) error { + return nil + }); err != nil { + t.Fatalf("step: %v", err) + } + if _, err := waylogv2.Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + r.assertParity("ok-simple.json") +} + +// --- Scenario 2: error-payment-cascade -------------------------------------- + +func TestParity_ErrorPaymentCascade(t *testing.T) { + r := newScenario(t) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: strings.Repeat("2", 32), + SpanID: strings.Repeat("2", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "POST", + "route": "/checkout", + "status": 200, // overwritten by SetHTTPStatus(502) below + }) + waylogv2.SetField(ctx, "user", map[string]any{"id": "u_123"}) + + if err := waylogv2.StepVoid(ctx, "db.load_cart", func(ctx context.Context) error { + return nil + }); err != nil { + t.Fatalf("db step: %v", err) + } + + wantErr := waylogv2.NewError("PMT_502", waylogv2.WithReason("upstream gateway 5xx")) + stepErr := waylogv2.StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + waylogv2.From(ctx).Warn("retrying payment", nil) + waylogv2.RecordOutgoingSpan(ctx, strings.Repeat("3", 16), "payment", "POST /charge") + waylogv2.Fail(ctx, wantErr) + return wantErr + }) + if !errors.Is(stepErr, wantErr) { + t.Fatalf("expected payment.charge to surface %v, got %v", wantErr, stepErr) + } + + waylogv2.SetHTTPStatus(ctx, 502) + + if _, err := waylogv2.Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + r.assertParity("error-payment-cascade.json") +} + +// --- Scenario 3: error-panic ------------------------------------------------ + +func TestParity_ErrorPanic(t *testing.T) { + r := newScenario(t) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: strings.Repeat("6", 32), + SpanID: strings.Repeat("7", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "POST", + "route": "/checkout", + "status": 500, + }) + if _, err := waylogv2.FinalizePanic(ctx); err != nil { + t.Fatalf("FinalizePanic: %v", err) + } + r.assertParity("error-panic.json") +} + +// --- Scenario 4: suppressed-healthcheck ------------------------------------- + +func TestParity_SuppressedHealthcheck(t *testing.T) { + r := newScenario(t) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: strings.Repeat("5", 32), + SpanID: strings.Repeat("6", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "GET", + "route": "/healthz", + "status": 200, + }) + waylogv2.Suppress(ctx) + if _, err := waylogv2.Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + r.assertParity("suppressed-healthcheck.json") +} + +// --- Scenario 5: aborted-cancel --------------------------------------------- + +func TestParity_AbortedCancel(t *testing.T) { + r := newScenario(t) + parent, cancel := context.WithCancel(context.Background()) + ctx := waylogv2.Begin(parent, waylogv2.BeginOptions{ + TraceID: strings.Repeat("4", 32), + SpanID: strings.Repeat("5", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "POST", + "route": "/checkout", + }) + cancel() + if _, err := waylogv2.FinalizeAborted(ctx); err != nil { + t.Fatalf("FinalizeAborted: %v", err) + } + r.assertParity("aborted-cancel.json") +} + +// --- Scenario 6: timeout-watchdog ------------------------------------------- +// +// The fixture encodes a watchdog firing while payment.charge is still +// active: the previous step closed normally, but the watchdog wins the +// seal before payment.charge can return. Reproducing that via public API +// means firing FinalizeTimeout from inside the active step's fn so the +// step is still on the stack at seal time. The SDK's flush-active-steps +// path then snapshots payment.charge into steps[] as the closed +// status-ok entry the fixture expects. + +func TestParity_TimeoutWatchdog(t *testing.T) { + r := newScenario(t) + ctx := waylogv2.Begin(context.Background(), waylogv2.BeginOptions{ + TraceID: strings.Repeat("3", 32), + SpanID: strings.Repeat("4", 16), + }) + waylogv2.SetField(ctx, "http", map[string]any{ + "method": "POST", + "route": "/checkout", + }) + + if err := waylogv2.StepVoid(ctx, "db.load_cart", func(ctx context.Context) error { + return nil + }); err != nil { + t.Fatalf("db.load_cart step: %v", err) + } + + if err := waylogv2.StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + // Fire the watchdog while still inside the step. Once Finalize + // seals the request, the surrounding StepVoid wrapper sees the + // sealed flag and skips its addStep — but the lifecycle path has + // already snapshotted payment.charge into steps[]. + if _, err := waylogv2.FinalizeTimeout(ctx); err != nil { + return err + } + return nil + }); err != nil { + t.Fatalf("payment.charge step: %v", err) + } + + r.assertParity("timeout-watchdog.json") +} diff --git a/pkg/waylog/v2/parity/runner.go b/pkg/waylog/v2/parity/runner.go new file mode 100644 index 0000000..5ab28bd --- /dev/null +++ b/pkg/waylog/v2/parity/runner.go @@ -0,0 +1,251 @@ +// Package parity provides a fixture-driven parity runner that drives the +// Waylog v2 SDK end-to-end through canonical scenarios and compares the +// emitted wide event against the golden fixtures under +// testdata/fixtures/v2/. +// +// The runner uses only the public Waylog SDK surface (Init, Shutdown, +// Begin, Step, Finalize*, From, …) so the test exercises the SDK exactly +// the way an external user would integrate it. +package parity + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" +) + +// fixtureDir is resolved relative to this source file at test time. +// pkg/waylog/v2/parity/ → ../../../../testdata/fixtures/v2/ +const fixtureDir = "../../../../testdata/fixtures/v2" + +// LoadFixture reads a golden v2 fixture by file name (e.g. "ok-simple.json"). +func LoadFixture(name string) ([]byte, error) { + path := filepath.Join(fixtureDir, name) + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("parity: load fixture %s: %w", name, err) + } + return raw, nil +} + +// MaskNonDeterministic returns a JSON blob with SDK-runtime-generated +// identity, timing, and span values replaced by stable sentinels so that +// pinned fixtures and live SDK output can be structurally compared. +// +// Masking targets: +// - root identity: event_id, trace_id, span_id, parent_span_id +// - root timing: ts_start, ts_end, duration_ms +// - per-step: span_id, start_ms, duration_ms +// - per-log: ts_offset_ms +// +// Empty-string identity values are masked to a separate sentinel so a +// fixture with an explicit `parent_span_id: ""` round-trips distinctly +// from a marshaler that drops the key entirely. +func MaskNonDeterministic(raw []byte) ([]byte, error) { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return nil, fmt.Errorf("parity: unmarshal for mask: %w", err) + } + masked := mask(v) + out, err := json.Marshal(masked) + if err != nil { + return nil, fmt.Errorf("parity: marshal masked: %w", err) + } + return out, nil +} + +// Equal reports whether two JSON blobs are structurally equivalent after +// masking and normalization (empty arrays / objects / strings collapse to +// nil so omitempty marshalling matches fixtures that explicitly set them). +func Equal(a, b []byte) (bool, error) { + maskedA, err := MaskNonDeterministic(a) + if err != nil { + return false, err + } + maskedB, err := MaskNonDeterministic(b) + if err != nil { + return false, err + } + var va, vb any + if err := json.Unmarshal(maskedA, &va); err != nil { + return false, fmt.Errorf("parity: unmarshal a: %w", err) + } + if err := json.Unmarshal(maskedB, &vb); err != nil { + return false, fmt.Errorf("parity: unmarshal b: %w", err) + } + return reflect.DeepEqual(normalize(va), normalize(vb)), nil +} + +const ( + sentinelPresent = "__MASKED__" + sentinelEmpty = "__MASKED_EMPTY__" + sentinelZeroInt = float64(0) // JSON numbers decode to float64 +) + +var rootStringMask = map[string]struct{}{ + "event_id": {}, + "trace_id": {}, + "span_id": {}, // also masks per-step span_id (handled in walkStep) + "parent_span_id": {}, + "ts_start": {}, + "ts_end": {}, +} + +var rootIntMask = map[string]struct{}{ + "duration_ms": {}, +} + +var stepIntMask = map[string]struct{}{ + "start_ms": {}, + "duration_ms": {}, +} + +func mask(node any) any { + root, ok := node.(map[string]any) + if !ok { + return node + } + out := make(map[string]any, len(root)) + for k, v := range root { + switch { + case isStringMask(k): + out[k] = stringMaskValue(v) + case isRootIntMask(k): + out[k] = sentinelZeroInt + case k == "steps": + out[k] = walkSteps(v) + case k == "logs": + out[k] = walkLogs(v) + default: + out[k] = v + } + } + return out +} + +func isStringMask(k string) bool { + _, ok := rootStringMask[k] + return ok +} + +func isRootIntMask(k string) bool { + _, ok := rootIntMask[k] + return ok +} + +func stringMaskValue(v any) any { + if s, ok := v.(string); ok && s == "" { + return sentinelEmpty + } + return sentinelPresent +} + +func walkSteps(v any) any { + arr, ok := v.([]any) + if !ok { + return v + } + out := make([]any, len(arr)) + for i, e := range arr { + out[i] = walkStep(e) + } + return out +} + +func walkStep(node any) any { + step, ok := node.(map[string]any) + if !ok { + return node + } + out := make(map[string]any, len(step)) + for k, v := range step { + switch { + case k == "span_id": + out[k] = stringMaskValue(v) + case isStepIntMask(k): + out[k] = sentinelZeroInt + default: + out[k] = v + } + } + return out +} + +func isStepIntMask(k string) bool { + _, ok := stepIntMask[k] + return ok +} + +func walkLogs(v any) any { + arr, ok := v.([]any) + if !ok { + return v + } + out := make([]any, len(arr)) + for i, e := range arr { + log, ok := e.(map[string]any) + if !ok { + out[i] = e + continue + } + entry := make(map[string]any, len(log)) + for k, val := range log { + if k == "ts_offset_ms" { + entry[k] = sentinelZeroInt + continue + } + entry[k] = val + } + out[i] = entry + } + return out +} + +// normalize collapses semantically-empty values (nil, "", [], {}) so that +// an omitempty struct field marshalling away matches a fixture that +// explicitly included an empty array or empty object. +func normalize(v any) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, vv := range x { + n := normalize(vv) + if isSemanticEmpty(n) { + continue + } + out[k] = n + } + if len(out) == 0 { + return nil + } + return out + case []any: + if len(x) == 0 { + return nil + } + out := make([]any, len(x)) + for i, vv := range x { + out[i] = normalize(vv) + } + return out + default: + return x + } +} + +func isSemanticEmpty(v any) bool { + switch x := v.(type) { + case nil: + return true + case string: + return x == "" + case map[string]any: + return len(x) == 0 + case []any: + return len(x) == 0 + default: + return false + } +} diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go index 4aac97d..6596938 100644 --- a/pkg/waylog/v2/request.go +++ b/pkg/waylog/v2/request.go @@ -231,6 +231,8 @@ type activeStep struct { name string spanID string downstream *eventv2.Downstream + startedAt time.Time + startMS int64 } type stepBuf struct { @@ -359,14 +361,13 @@ func (r *request) setHTTPStatusLocked(status int) { httpMap["status"] = status } -func (r *request) markLifecycleLocked(status eventv2.Status, code, reason string) { +func (r *request) markLifecycleLocked(status eventv2.Status, code string) { if r.suppressed { return } r.finalStatus = status r.anchorStep = r.activeStepLocked() r.anchorCode = code - r.recordErrorLocked(code, reason) } // degradeToHeaderOnlyLocked discards buffered detail and switches the request diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index b94dc66..e67141e 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -248,7 +248,8 @@ func TestFinalizePanicSynthesizesReservedLifecycleCode(t *testing.T) { h := newHarness(t, Config{}) ctx := Begin(context.Background(), BeginOptions{}) r := requestFromContext(ctx) - r.pushStep("payment.charge") + now := time.Now() + r.pushStep("payment.charge", now, int64(now.Sub(r.tsStart)/1e6)) _, _ = FinalizePanic(ctx) diff --git a/pkg/waylog/v2/step.go b/pkg/waylog/v2/step.go index 9d5c23b..56ae6da 100644 --- a/pkg/waylog/v2/step.go +++ b/pkg/waylog/v2/step.go @@ -18,10 +18,9 @@ func Step[T any](ctx context.Context, name string, fn func(ctx context.Context) if r == nil || name == "" { return fn(ctx) } - r.pushStep(name) - startedAt := time.Now() startMS := int64(startedAt.Sub(r.tsStart) / 1e6) + r.pushStep(name, startedAt, startMS) v, err := fn(ctx) dur := int64(time.Since(startedAt) / 1e6) @@ -99,9 +98,13 @@ func Suppress(ctx context.Context) { r.anchorCode = "" } -func (r *request) pushStep(name string) { +func (r *request) pushStep(name string, startedAt time.Time, startMS int64) { r.mu.Lock() - r.stepStack = append(r.stepStack, activeStep{name: name}) + r.stepStack = append(r.stepStack, activeStep{ + name: name, + startedAt: startedAt, + startMS: startMS, + }) r.mu.Unlock() } diff --git a/scripts/bench-gate.sh b/scripts/bench-gate.sh new file mode 100755 index 0000000..927641b --- /dev/null +++ b/scripts/bench-gate.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# bench-gate.sh — Phase 1a regression gate for the Go v2 SDK. +# +# This gate is a *baseline regression* check, not proof that the §4.4.1 +# performance budgets are met. Time thresholds match the spec budgets + +# ~10% slack and pass comfortably today. Allocation thresholds are +# current-baseline + 10% — middleware and the 20-step / 50-log assemble +# path do not yet meet the §4.4.1 alloc targets, and that gap is tracked +# as Phase 1a perf debt in docs/v2-plan.md. The gate's job here is to +# prevent accidental further regressions while the team prioritizes TS +# parity and the runtime bridge. +# +# When the SDK is optimized to meet §4.4.1 alloc targets, tighten the +# budget rows below and remove the perf-debt note in docs/v2-plan.md. +# +# Usage: +# ./scripts/bench-gate.sh # one go test -bench run +# BENCH_TIME=2s ./scripts/bench-gate.sh +# +# Output: +# - prints raw bench output to stderr +# - prints `: NS= ALLOCS=` per benchmark to stdout +# - exits 0 with `BENCH GATE PASSED` on success +# - exits 1 with the first failing budget on regression + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +BENCH_TIME="${BENCH_TIME:-1s}" +PKG="./pkg/waylog/v2/bench/..." + +# Budget table: NAME NS_OP_MAX ALLOCS_OP_MAX +# +# Time budgets follow §4.4.1 (spec) + 10% slack. +# Alloc budgets for MiddlewareNoOp and Assemble20Steps50Logs follow +# observed-baseline + 10% (regression protection only); the §4.4.1 alloc +# targets are 16 / steady-state for middleware and a tighter assemble +# number, both of which remain open perf debt. +budgets=( + "BenchmarkMiddlewareNoOp 33000 45" + "BenchmarkStepEmptyBody 5500 4" + "BenchmarkLoggerInfo 3300 3" + "BenchmarkAssemble20Steps50Logs 110000 360" +) + +bench_out="$(mktemp)" +trap 'rm -f "$bench_out"' EXIT + +echo "running: go test -bench=. -benchmem -run='^$' -benchtime=${BENCH_TIME} ${PKG}" >&2 +if ! go test -bench=. -benchmem -run='^$' -benchtime="$BENCH_TIME" "$PKG" >"$bench_out" 2>&1; then + cat "$bench_out" >&2 + echo "BENCH GATE FAILED: go test -bench failed" >&2 + exit 1 +fi + +cat "$bench_out" >&2 + +failed=0 + +check() { + local name="$1" ns_max="$2" allocs_max="$3" + # Match a Benchmark line; capture ns/op and allocs/op fields. + # Example line: + # BenchmarkStepEmptyBody-8 12740458 91.63 ns/op 0 B/op 0 allocs/op + local line + line="$(awk -v n="$name" '$1 ~ "^"n"(-[0-9]+)?$" {print; exit}' "$bench_out")" + if [[ -z "$line" ]]; then + echo "BENCH GATE FAILED: ${name} not found in bench output" >&2 + failed=1 + return + fi + + # ns/op: token preceding "ns/op" + local ns_op + ns_op="$(awk -v n="$name" ' + $1 ~ "^"n"(-[0-9]+)?$" { + for (i = 1; i <= NF; i++) { + if ($i == "ns/op") { print $(i-1); exit } + } + } + ' "$bench_out")" + + local allocs_op + allocs_op="$(awk -v n="$name" ' + $1 ~ "^"n"(-[0-9]+)?$" { + for (i = 1; i <= NF; i++) { + if ($i == "allocs/op") { print $(i-1); exit } + } + } + ' "$bench_out")" + + if [[ -z "$ns_op" || -z "$allocs_op" ]]; then + echo "BENCH GATE FAILED: ${name} bench line missing fields: ${line}" >&2 + failed=1 + return + fi + + echo "${name}: NS=${ns_op} ALLOCS=${allocs_op}" + + # Use awk for floating-point comparison on ns_op. + if awk -v v="$ns_op" -v m="$ns_max" 'BEGIN { exit !(v+0 > m+0) }'; then + echo "BENCH GATE FAILED: ${name} ${ns_op} ns/op > budget ${ns_max} ns/op" >&2 + failed=1 + fi + if (( allocs_op > allocs_max )); then + echo "BENCH GATE FAILED: ${name} ${allocs_op} allocs/op > budget ${allocs_max} allocs/op" >&2 + failed=1 + fi +} + +for entry in "${budgets[@]}"; do + # shellcheck disable=SC2086 + check $entry +done + +if (( failed )); then + exit 1 +fi + +echo "BENCH GATE PASSED" From 4795ff37edbbe4cc75d14e1164bbf7e9ec72da4e Mon Sep 17 00:00:00 2001 From: skota-hash Date: Sun, 26 Apr 2026 20:00:16 -0400 Subject: [PATCH 7/8] feat(sdk): completed TypeScript v2 SDK parity and harden transport/adapters Rebuild the TypeScript SDK around schema 2.0 and the Phase 1 contract: - replace the schema 1.1 TS event surface with schema 2.0 wide-event types - add v2 core APIs: init, shutdown, stats, begin, finalize, lifecycle finalizers, from, step, stepSync, fail, newError, suppress, explain - implement AsyncLocalStorage-backed request context with standalone Context fallback - add request buffering for fields, steps, logs, errors, anchor, active-step stack, caps, redaction, rate limiting, and exactly-once finalization - match Go lifecycle semantics for suppression, panic, aborted, timeout, and active-step timeout snapshots - add local explain output and dev dual-emit for per-log pretty lines plus final pretty JSON Add TypeScript v2 delivery transport: - support ingestUrl delivery to /v1/events - keep NDJSON batching as the default transport path - add explicit single-event JSON mode via batchMode=false - add ok/priority queue partitioning so error/timeout/partial/aborted events are preserved ahead of ok/suppressed events - add maxInFlightBytes pressure handling, retry/backoff, Retry-After handling, shutdown drain, and delivery/drop stats - fix shutdown so retryable batches are not dropped immediately during drain - fix ingest URL normalization so query strings are preserved while /v1/events is applied to the pathname Update TypeScript framework adapters: - rewrite Express and Hono adapters to use the v2 core lifecycle - add Next.js and NestJS entrypoints and package exports - capture trace propagation, HTTP method/route/status, thrown errors, 5xx failures, watchdog timeout, and suppression behavior - fix Express synchronous throw handling so finalized requests do not later inflate late-completion stats Add conformance and parity coverage: - replace old schema 1.1 TS tests with v2 lifecycle, transport, adapter, and fixture parity suites - validate all six shared v2 fixture scenarios from testdata/fixtures/v2 - cover transport auth, NDJSON, JSON mode, retry, permanent drops, pressure behavior, and URL normalization - cover Express/Hono/Next/Nest adapter smoke paths Update examples and docs: - refresh README TypeScript usage to the v2 middleware/useLogger surface - update the TS e2e smoke driver to emit a deterministic schema 2.0 payment-failure request - clarify that the low-level e2e driver is not the recommended user-facing integration path Verification: - cd packages/waylog-ts && npm run build - cd packages/waylog-ts && npm test - go test -bench='BenchmarkStepEmptyBody|BenchmarkLoggerInfo' -benchmem ./pkg/waylog/v2/bench -run='^$' --- README.md | 6 +- packages/waylog-ts/examples/e2e-emit.ts | 58 +- packages/waylog-ts/package.json | 8 + .../waylog-ts/src/__tests__/adapters.test.ts | 111 +++ packages/waylog-ts/src/__tests__/core.test.ts | 177 +++++ .../waylog-ts/src/__tests__/error.test.ts | 39 - .../waylog-ts/src/__tests__/express.test.ts | 74 -- packages/waylog-ts/src/__tests__/hono.test.ts | 74 -- .../waylog-ts/src/__tests__/logger.test.ts | 114 --- .../waylog-ts/src/__tests__/parity.test.ts | 124 +++ .../waylog-ts/src/__tests__/transport.test.ts | 147 ++-- packages/waylog-ts/src/error.ts | 35 +- packages/waylog-ts/src/express.ts | 102 ++- packages/waylog-ts/src/hono.ts | 82 +- packages/waylog-ts/src/index.ts | 58 +- packages/waylog-ts/src/logger.ts | 719 ++++++++++++++---- packages/waylog-ts/src/nest.ts | 20 + packages/waylog-ts/src/next.ts | 40 + packages/waylog-ts/src/transport.ts | 319 ++++++-- packages/waylog-ts/src/types.ts | 210 ++--- 20 files changed, 1741 insertions(+), 776 deletions(-) create mode 100644 packages/waylog-ts/src/__tests__/adapters.test.ts create mode 100644 packages/waylog-ts/src/__tests__/core.test.ts delete mode 100644 packages/waylog-ts/src/__tests__/error.test.ts delete mode 100644 packages/waylog-ts/src/__tests__/express.test.ts delete mode 100644 packages/waylog-ts/src/__tests__/hono.test.ts delete mode 100644 packages/waylog-ts/src/__tests__/logger.test.ts create mode 100644 packages/waylog-ts/src/__tests__/parity.test.ts create mode 100644 packages/waylog-ts/src/nest.ts create mode 100644 packages/waylog-ts/src/next.ts diff --git a/README.md b/README.md index fc58c68..7294f1c 100644 --- a/README.md +++ b/README.md @@ -57,17 +57,17 @@ import { waylog, useLogger } from "@waylog/sdk/express"; app.use(waylog({ service: "checkout", env: "prod", - endpoint: "http://localhost:8080", + ingestUrl: "http://localhost:8080", apiKey: process.env.WAYLOG_WRITE_KEY, })); app.post("/buy", (req, res) => { - useLogger(req).set({ user: { id: req.user.id, tier: "vip" } }); + useLogger(req).info("cart loaded", { user_id: req.user.id, tier: "vip" }); res.sendStatus(200); }); ``` -`@waylog/sdk` is ESM-only, Node 18+, and ships Express and Hono middleware (`@waylog/sdk/express`, `@waylog/sdk/hono`). `createLogger().withRequest().set().emit()` is also exposed for non-middleware use. +`@waylog/sdk` is ESM-only, Node 18+, and ships standalone core APIs plus Express, Hono, Next.js, and NestJS entrypoints (`@waylog/sdk/express`, `@waylog/sdk/hono`, `@waylog/sdk/next`, `@waylog/sdk/nest`). ### Go SDK diff --git a/packages/waylog-ts/examples/e2e-emit.ts b/packages/waylog-ts/examples/e2e-emit.ts index f412e25..5940509 100644 --- a/packages/waylog-ts/examples/e2e-emit.ts +++ b/packages/waylog-ts/examples/e2e-emit.ts @@ -1,36 +1,36 @@ -// e2e-emit.ts emits a single TS-SDK trace (1 request) to the ingest server -// and prints the trace_id on stdout. Used by scripts/e2e-mark2.sh to drive -// the TypeScript SDK path. -// -// Runner: vite-node (available in local node_modules/.bin after `npm install` -// inside packages/waylog-ts — ships with vitest). +// e2e-emit.ts is a low-level schema 2.0 smoke driver, not the recommended +// user-facing integration example. Real apps should usually use framework +// middleware (`@waylog/sdk/express`, `@waylog/sdk/hono`, etc.) and +// `useLogger(...)`. This file exercises the standalone core directly so the +// e2e path can emit a deterministic payment-failure event without depending +// on any framework adapter. -import { createLogger } from "../src/index.js"; +import { begin, fail, finalize, from, init, newError, recordOutgoingSpan, setField, setHTTPStatus, stepSync, traceId } from "../src/index.js"; -const endpoint = process.env.INGEST_URL ?? "http://localhost:8080"; - -const logger = createLogger({ - endpoint, +init({ + ingestUrl: process.env.INGEST_URL ?? "http://localhost:8080", service: "ts-e2e", env: "dev", version: "0.1.0", - batchSize: 1, - flushIntervalMs: 100, -}); - -const req = logger.withRequest({ - flow: "purchase", - user: { id: "ts-e2e-user", tier: "standard", region: "us-east-1" }, - httpMethod: "GET", - routeTemplate: "/api/e2e", }); -// Extract the trace_id from the outbound traceparent: "00---". -const traceId = req.traceparent().split("-")[1]!; - -req.emit({ success: true, status_code: 200 }); - -await logger.flush(); -await logger.close(); - -console.log(traceId); +const ctx = begin({}); +setField(ctx, "http", { method: "POST", route: "/api/e2e", status: 200 }); +setField(ctx, "user", { id: "ts-e2e-user", tier: "standard", region: "us-east-1" }); + +stepSync(ctx, "db.load_cart", () => undefined); +const err = newError("PMT_502", { reason: "upstream gateway 5xx" }); +try { + stepSync(ctx, "payment.charge", () => { + from(ctx).warn("retrying payment"); + recordOutgoingSpan(ctx, "3333333333333333", "payment", "POST /charge"); + fail(ctx, err); + throw err; + }); +} catch { + setHTTPStatus(ctx, 502); +} + +await finalize(ctx); + +console.log(traceId(ctx)); diff --git a/packages/waylog-ts/package.json b/packages/waylog-ts/package.json index b90604a..5af51cc 100644 --- a/packages/waylog-ts/package.json +++ b/packages/waylog-ts/package.json @@ -17,6 +17,14 @@ "./hono": { "types": "./dist/hono.d.ts", "import": "./dist/hono.js" + }, + "./next": { + "types": "./dist/next.d.ts", + "import": "./dist/next.js" + }, + "./nest": { + "types": "./dist/nest.d.ts", + "import": "./dist/nest.js" } }, "files": [ diff --git a/packages/waylog-ts/src/__tests__/adapters.test.ts b/packages/waylog-ts/src/__tests__/adapters.test.ts new file mode 100644 index 0000000..7d2d8f1 --- /dev/null +++ b/packages/waylog-ts/src/__tests__/adapters.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it, vi } from "vitest"; +import { middleware as expressMiddleware, useLogger as useExpressLogger } from "../express.js"; +import { middleware as honoMiddleware, useLogger as useHonoLogger } from "../hono.js"; +import { middleware as nestMiddleware } from "../nest.js"; +import { middleware as nextMiddleware } from "../next.js"; +import { stats } from "../index.js"; +import type { WideEvent } from "../types.js"; + +function outputCapture() { + const lines: string[] = []; + return { + config: { service: "web", env: "test", output: (line: string) => lines.push(line) }, + event(): WideEvent { + const raw = lines.find((line) => line.trim().startsWith("{")); + if (!raw) throw new Error("no event"); + return JSON.parse(raw) as WideEvent; + }, + }; +} + +function mockExpress(headers: Record = {}) { + const listeners: Record void>> = {}; + const req = { method: "GET", url: "/checkout/42", route: { path: "/checkout/:id" }, headers }; + const res = { + statusCode: 200, + setHeader: vi.fn(), + on: (ev: "finish" | "close", cb: () => void) => { + (listeners[ev] ??= []).push(cb); + }, + trigger: (ev: "finish" | "close") => listeners[ev]?.forEach((cb) => cb()), + }; + return { req, res }; +} + +describe("TS adapters", () => { + it("Express captures route, trace, status, and logger fields", async () => { + const h = outputCapture(); + const mw = expressMiddleware(h.config); + const { req, res } = mockExpress({ traceparent: `00-${"a".repeat(32)}-${"b".repeat(16)}-01` }); + mw(req as any, res as any, () => { + useExpressLogger(req as any).info("hello", { user_id: "u_1" }); + }); + res.trigger("finish"); + const ev = h.event(); + expect(ev.trace_id).toBe("a".repeat(32)); + expect((ev.fields?.http as any).route).toBe("/checkout/:id"); + expect(ev.logs?.[0]?.fields).toEqual({ user_id: "u_1" }); + }); + + it("Express marks 5xx as failures", () => { + const h = outputCapture(); + const mw = expressMiddleware(h.config); + const { req, res } = mockExpress(); + mw(req as any, res as any, () => undefined); + res.statusCode = 503; + res.trigger("finish"); + expect(h.event().anchor?.error_code).toBe("HTTP_503"); + }); + + it("Express thrown errors finalize once", () => { + const h = outputCapture(); + const mw = expressMiddleware(h.config); + const { req, res } = mockExpress(); + expect(() => mw(req as any, res as any, () => { + throw new Error("boom"); + })).toThrow("boom"); + res.trigger("finish"); + res.trigger("close"); + expect(h.event().status).toBe("error"); + expect(stats().lateCompletionAfterEmit).toBe(0); + }); + + it("Hono captures thrown errors", async () => { + const h = outputCapture(); + const store = new Map(); + const c = { + req: { method: "POST", routePath: "/buy/:id", path: "/buy/42", header: () => undefined }, + res: { headers: { set: vi.fn() }, status: 200 }, + set: (k: string, v: unknown) => store.set(k, v), + get: (k: string) => store.get(k), + }; + const mw = honoMiddleware(h.config); + await expect(mw(c as any, async () => { + useHonoLogger(c as any).warn("before boom"); + throw new Error("boom"); + })).rejects.toThrow("boom"); + const ev = h.event(); + expect(ev.status).toBe("error"); + expect((ev.fields?.http as any).status).toBe(500); + }); + + it("Next wrapper finalizes handler output", async () => { + const h = outputCapture(); + const handler = nextMiddleware(h.config, async (_req, ctx) => { + return { status: 201, ctx }; + }); + const res = await handler({ method: "GET", url: "http://local/items", headers: { get: () => null } }); + expect(res.status).toBe(201); + expect((h.event().fields?.http as any).route).toBe("/items"); + }); + + it("Nest middleware reuses Express lifecycle semantics", () => { + const h = outputCapture(); + const mw = nestMiddleware(h.config); + const { req, res } = mockExpress(); + mw(req as any, res as any, () => undefined); + res.statusCode = 204; + res.trigger("finish"); + expect((h.event().fields?.http as any).status).toBe(204); + }); +}); diff --git a/packages/waylog-ts/src/__tests__/core.test.ts b/packages/waylog-ts/src/__tests__/core.test.ts new file mode 100644 index 0000000..f50d172 --- /dev/null +++ b/packages/waylog-ts/src/__tests__/core.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from "vitest"; +import { + begin, + explain, + fail, + finalize, + finalizeAborted, + finalizePanic, + finalizeTimeout, + from, + init, + newError, + parseTraceparent, + recordOutgoingSpan, + setField, + setHTTPStatus, + shutdown, + stats, + stepSync, + suppress, +} from "../index.js"; +import type { WideEvent } from "../types.js"; + +function harness() { + const lines: string[] = []; + init({ service: "checkout", env: "test", output: (line) => lines.push(line) }); + return { + lines, + event(): WideEvent { + const json = lines.filter((line) => line.trim().startsWith("{")).at(-1); + if (!json) throw new Error("no event emitted"); + return JSON.parse(json) as WideEvent; + }, + }; +} + +describe("v2 core lifecycle", () => { + it("emits an ok event with steps and fields", async () => { + const h = harness(); + const ctx = begin({}, { traceId: "1".repeat(32), spanId: "1".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout", status: 200 }); + stepSync(ctx, "db.load_cart", () => undefined); + await finalize(ctx); + + const ev = h.event(); + expect(ev.schema_version).toBe("2.0"); + expect(ev.status).toBe("ok"); + expect(ev.steps?.[0]?.name).toBe("db.load_cart"); + expect((ev.fields?.http as any).route).toBe("/checkout"); + }); + + it("records explicit failures and local explain output", async () => { + harness(); + const ctx = begin({}, { traceId: "2".repeat(32), spanId: "2".repeat(16) }); + const err = newError("PMT_502", { reason: "upstream gateway 5xx" }); + expect(() => stepSync(ctx, "payment.charge", () => { + fail(ctx, err); + throw err; + })).toThrow("upstream gateway 5xx"); + setHTTPStatus(ctx, 502); + const explained = await explain(ctx); + await finalize(ctx); + + expect(explained.anchor?.error_code).toBe("PMT_502"); + expect(explained.toString()).toContain("PMT_502"); + }); + + it("suppressed events keep fields but drop detail", async () => { + const h = harness(); + const ctx = begin({}); + setField(ctx, "http", { method: "GET", route: "/healthz", status: 200 }); + suppress(ctx); + await finalize(ctx); + + const ev = h.event(); + expect(ev.status).toBe("suppressed"); + expect(ev.fields?.http).toEqual({ method: "GET", route: "/healthz", status: 200 }); + expect(ev.steps).toBeUndefined(); + }); + + it("panic records errors[] while timeout and abort only anchor", async () => { + const panicH = harness(); + const panicCtx = begin({}); + await finalizePanic(panicCtx); + expect(panicH.event().errors?.[0]?.code).toBe("WAYLOG_PANIC"); + + const timeoutH = harness(); + const timeoutCtx = begin({}); + await finalizeTimeout(timeoutCtx); + expect(timeoutH.event().anchor?.error_code).toBe("WAYLOG_TIMEOUT"); + expect(timeoutH.event().errors).toBeUndefined(); + + const abortedH = harness(); + const abortedCtx = begin({}); + await finalizeAborted(abortedCtx); + expect(abortedH.event().anchor?.error_code).toBe("WAYLOG_ABORTED"); + expect(abortedH.event().errors).toBeUndefined(); + }); + + it("timeout snapshots an active step", async () => { + const h = harness(); + const ctx = begin({}); + stepSync(ctx, "payment.charge", () => { + recordOutgoingSpan(ctx, "3".repeat(16), "payment", "POST /charge"); + void finalizeTimeout(ctx); + }); + + const ev = h.event(); + expect(ev.status).toBe("timeout"); + expect(ev.anchor?.step).toBe("payment.charge"); + expect(ev.steps?.[0]?.name).toBe("payment.charge"); + expect(ev.steps?.[0]?.downstream?.service).toBe("payment"); + }); + + it("rate limiting drops ok before priority", async () => { + const lines: string[] = []; + init({ service: "checkout", env: "test", output: (line) => lines.push(line), maxEventsPerSec: 1 }); + await finalize(begin({})); + await finalize(begin({})); + const failed = begin({}); + fail(failed, newError("BOOM")); + await finalize(failed); + expect(lines.filter((line) => line.trim().startsWith("{"))).toHaveLength(2); + expect(stats().eventsDropped).toBe(1); + }); + + it("dev mode emits pretty log lines and final JSON", async () => { + const lines: string[] = []; + init({ service: "checkout", env: "dev", output: (line) => lines.push(line) }); + const ctx = begin({}); + from(ctx).warn("retrying payment", { cart_id: "c_1" }); + await finalize(ctx); + expect(lines.some((line) => line.includes("[WARN] retrying payment"))).toBe(true); + expect(lines.some((line) => line.includes("\n \"schema_version\""))).toBe(true); + }); + + it("redacts final fields synchronously", async () => { + const h = (() => { + const lines: string[] = []; + init({ + service: "checkout", + env: "test", + output: (line) => lines.push(line), + redactor: (fields) => ({ ...fields, token: "[REDACTED]" }), + }); + return { event: () => JSON.parse(lines.find((line) => line.trim().startsWith("{"))!) as WideEvent }; + })(); + const ctx = begin({}); + setField(ctx, "token", "secret"); + await finalize(ctx); + expect(h.event().fields?.token).toBe("[REDACTED]"); + }); + + it("caps buffered logs without rejecting request finalization", async () => { + const h = (() => { + const lines: string[] = []; + init({ service: "checkout", env: "test", output: (line) => lines.push(line), maxLogs: 1 }); + return { event: () => JSON.parse(lines.find((line) => line.trim().startsWith("{"))!) as WideEvent }; + })(); + const ctx = begin({}); + from(ctx).info("one"); + from(ctx).info("two"); + await finalize(ctx); + expect(h.event().logs).toHaveLength(1); + expect(stats().logsDropped).toBe(1); + }); +}); + +describe("traceparent", () => { + it("parses valid W3C traceparent", () => { + expect(parseTraceparent(`00-${"a".repeat(32)}-${"b".repeat(16)}-01`)).toEqual({ traceId: "a".repeat(32), spanId: "b".repeat(16) }); + }); + + it("rejects invalid traceparent", () => { + expect(parseTraceparent("bad")).toBeUndefined(); + }); +}); diff --git a/packages/waylog-ts/src/__tests__/error.test.ts b/packages/waylog-ts/src/__tests__/error.test.ts deleted file mode 100644 index 056ce26..0000000 --- a/packages/waylog-ts/src/__tests__/error.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { createError, isErrorContext } from "../error.js"; - -describe("createError", () => { - it("requires code and message", () => { - expect(() => createError({ code: "", message: "x" })).toThrow(); - expect(() => createError({ code: "PMT_500", message: "" })).toThrow(); - }); - - it("omits path and reason when not supplied", () => { - const e = createError({ code: "PMT_500", message: "boom" }); - expect(e).toEqual({ code: "PMT_500", message: "boom" }); - }); - - it("includes path and reason when supplied", () => { - const e = createError({ - code: "PMT_502", - message: "upstream", - path: "stripe.charge", - reason: "rate_limited", - }); - expect(e).toEqual({ - code: "PMT_502", - message: "upstream", - path: "stripe.charge", - reason: "rate_limited", - }); - }); -}); - -describe("isErrorContext", () => { - it("accepts valid shape", () => { - expect(isErrorContext({ code: "x", message: "y" })).toBe(true); - }); - it("rejects Error instance and string", () => { - expect(isErrorContext(new Error("x"))).toBe(false); - expect(isErrorContext("boom")).toBe(false); - }); -}); diff --git a/packages/waylog-ts/src/__tests__/express.test.ts b/packages/waylog-ts/src/__tests__/express.test.ts deleted file mode 100644 index 2437927..0000000 --- a/packages/waylog-ts/src/__tests__/express.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { useLogger, waylog } from "../express.js"; -import type { WideEvent } from "../types.js"; - -// Synthetic Express-compatible req/res just rich enough to exercise the -// middleware. No actual express dependency. -function mockReqRes(headers: Record = {}) { - const listeners: Record void>> = {}; - const res = { - statusCode: 200, - setHeader: vi.fn(), - on: (ev: string, cb: () => void) => { - (listeners[ev] ??= []).push(cb); - }, - trigger: (ev: string) => (listeners[ev] || []).forEach((cb) => cb()), - }; - const req = { - method: "GET", - url: "/checkout", - route: { path: "/checkout" }, - headers, - }; - return { req, res }; -} - -describe("express middleware", () => { - it("attaches a RequestLogger, emits on response finish, and sets traceparent", async () => { - const captured: WideEvent[] = []; - const fakeFetch = vi.fn(async (_u: string, init?: RequestInit) => { - const parsed = JSON.parse(init?.body as string); - if (Array.isArray(parsed)) captured.push(...parsed); - else captured.push(parsed); - return new Response("ok"); - }) as unknown as typeof fetch; - - const mw = waylog({ endpoint: "http://x", service: "web", batchSize: 1, fetch: fakeFetch }); - const { req, res } = mockReqRes({ traceparent: `00-${"a".repeat(32)}-${"b".repeat(16)}-01` }); - const next = vi.fn(); - mw(req as any, res as any, next); - expect(next).toHaveBeenCalled(); - expect(res.setHeader).toHaveBeenCalledWith("traceparent", expect.stringMatching(/^00-a{32}-[0-9a-f]{16}-01$/)); - - const log = useLogger(req as any); - log.set({ user: { id: "u9", tier: "pro", region: "us", vip: false } }); - res.statusCode = 200; - res.trigger("finish"); - await vi.waitFor(() => expect(captured).toHaveLength(1)); - expect(captured[0]!.request.trace_id).toBe("a".repeat(32)); - expect(captured[0]!.user.id).toBe("u9"); - expect(captured[0]!.outcome.status_code).toBe(200); - expect(captured[0]!.outcome.success).toBe(true); - }); - - it("marks 5xx as failures", async () => { - const captured: WideEvent[] = []; - const fakeFetch = vi.fn(async (_u: string, init?: RequestInit) => { - captured.push(JSON.parse(init?.body as string)); - return new Response("ok"); - }) as unknown as typeof fetch; - const mw = waylog({ endpoint: "http://x", service: "web", batchSize: 1, fetch: fakeFetch }); - const { req, res } = mockReqRes(); - mw(req as any, res as any, () => {}); - res.statusCode = 503; - res.trigger("finish"); - await vi.waitFor(() => expect(captured).toHaveLength(1)); - expect(captured[0]!.outcome.success).toBe(false); - expect(captured[0]!.outcome.status_code).toBe(503); - }); - - it("useLogger throws when middleware was not registered", () => { - const req = { headers: {} } as any; - expect(() => useLogger(req)).toThrow(/middleware/); - }); -}); diff --git a/packages/waylog-ts/src/__tests__/hono.test.ts b/packages/waylog-ts/src/__tests__/hono.test.ts deleted file mode 100644 index fe174e1..0000000 --- a/packages/waylog-ts/src/__tests__/hono.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { useLogger, waylog } from "../hono.js"; -import type { WideEvent } from "../types.js"; - -function mockCtx(headers: Record = {}) { - const store = new Map(); - const resHeaders = new Map(); - const c = { - req: { - method: "POST", - routePath: "/checkout", - path: "/checkout", - header: (n: string) => headers[n.toLowerCase()], - }, - res: { - headers: { set: (n: string, v: string) => resHeaders.set(n, v) }, - status: 200, - }, - set: (k: string, v: unknown) => store.set(k, v), - get: (k: string) => store.get(k), - }; - return { c, store, resHeaders }; -} - -describe("hono middleware", () => { - it("attaches logger, surfaces traceparent, and emits after next()", async () => { - const captured: WideEvent[] = []; - const fakeFetch = vi.fn(async (_u: string, init?: RequestInit) => { - captured.push(JSON.parse(init?.body as string)); - return new Response("ok"); - }) as unknown as typeof fetch; - - const mw = waylog({ endpoint: "http://x", service: "web", batchSize: 1, fetch: fakeFetch }); - const { c, resHeaders } = mockCtx({ traceparent: `00-${"a".repeat(32)}-${"b".repeat(16)}-01` }); - - await mw(c as any, async () => { - useLogger(c as any).set({ request: { flow: "checkout" } }); - (c.res as any).status = 201; - }); - - expect(resHeaders.get("traceparent")).toMatch(/^00-a{32}-[0-9a-f]{16}-01$/); - await vi.waitFor(() => expect(captured).toHaveLength(1)); - expect(captured[0]!.request.trace_id).toBe("a".repeat(32)); - expect(captured[0]!.outcome.status_code).toBe(201); - expect(captured[0]!.outcome.success).toBe(true); - }); - - it("marks thrown errors as failures and rethrows", async () => { - const captured: WideEvent[] = []; - const fakeFetch = vi.fn(async (_u: string, init?: RequestInit) => { - captured.push(JSON.parse(init?.body as string)); - return new Response("ok"); - }) as unknown as typeof fetch; - - const mw = waylog({ endpoint: "http://x", service: "web", batchSize: 1, fetch: fakeFetch }); - const { c } = mockCtx(); - - await expect( - mw(c as any, async () => { - throw new Error("boom"); - }), - ).rejects.toThrow("boom"); - - await vi.waitFor(() => expect(captured).toHaveLength(1)); - expect(captured[0]!.outcome.success).toBe(false); - expect(captured[0]!.outcome.status_code).toBe(500); - expect(captured[0]!.error?.message).toBe("boom"); - }); - - it("useLogger throws when middleware was not registered", () => { - const { c } = mockCtx(); - expect(() => useLogger(c as any)).toThrow(/middleware/); - }); -}); diff --git a/packages/waylog-ts/src/__tests__/logger.test.ts b/packages/waylog-ts/src/__tests__/logger.test.ts deleted file mode 100644 index 37f3bf7..0000000 --- a/packages/waylog-ts/src/__tests__/logger.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { createLogger, newSpanId, newTraceId, parseTraceparent } from "../logger.js"; -import type { WideEvent } from "../types.js"; - -function captureLogger() { - const captured: WideEvent[] = []; - const fakeFetch = vi.fn(async (_url: string, init?: RequestInit) => { - const body = init?.body as string; - const parsed = JSON.parse(body); - if (Array.isArray(parsed)) captured.push(...(parsed as WideEvent[])); - else captured.push(parsed as WideEvent); - return new Response("ok"); - }) as unknown as typeof fetch; - const logger = createLogger({ - endpoint: "http://test", - service: "svc", - version: "1.0.0", - env: "test", - batchSize: 1, - fetch: fakeFetch, - }); - return { logger, captured }; -} - -describe("trace IDs", () => { - it("newTraceId returns 32 hex chars", () => { - expect(newTraceId()).toMatch(/^[0-9a-f]{32}$/); - }); - it("newSpanId returns 16 hex chars", () => { - expect(newSpanId()).toMatch(/^[0-9a-f]{16}$/); - }); -}); - -describe("parseTraceparent", () => { - it("parses a well-formed header", () => { - const h = `00-${"a".repeat(32)}-${"b".repeat(16)}-01`; - expect(parseTraceparent(h)).toEqual({ traceId: "a".repeat(32), spanId: "b".repeat(16) }); - }); - it("rejects wrong version, lengths, and non-hex", () => { - expect(parseTraceparent(undefined)).toBeUndefined(); - expect(parseTraceparent("01-x-y-z")).toBeUndefined(); - expect(parseTraceparent(`00-${"a".repeat(30)}-${"b".repeat(16)}-01`)).toBeUndefined(); - expect(parseTraceparent(`00-${"z".repeat(32)}-${"b".repeat(16)}-01`)).toBeUndefined(); - }); -}); - -describe("RequestLogger lifecycle", () => { - it("emit posts a valid WideEvent with default success event_name", async () => { - const { logger, captured } = captureLogger(); - const r = logger.withRequest({ traceId: "a".repeat(32) }); - r.set({ user: { id: "u1", tier: "pro", region: "eu", vip: true }, request: { flow: "checkout" } }); - r.emit(); - await logger.flush(); - - expect(captured).toHaveLength(1); - const ev = captured[0]!; - expect(ev.schema_version).toBe("1.1"); - expect(ev.event_name).toBe("svc.request"); - expect(ev.outcome.success).toBe(true); - expect(ev.user.id).toBe("u1"); - expect(ev.request.trace_id).toBe("a".repeat(32)); - }); - - it("error() + emit() produces svc.error with structured error", async () => { - const { logger, captured } = captureLogger(); - const r = logger.withRequest(); - r.error({ code: "PMT_502", message: "upstream timed out", path: "stripe.charge", reason: "rate_limited" }); - r.emit({ success: false, status_code: 502, kind: "http" }); - await logger.flush(); - - const ev = captured[0]!; - expect(ev.event_name).toBe("svc.error"); - expect(ev.outcome.success).toBe(false); - expect(ev.outcome.status_code).toBe(502); - expect(ev.error?.code).toBe("PMT_502"); - expect(ev.error?.path).toBe("stripe.charge"); - }); - - it("emit is idempotent", async () => { - const { logger, captured } = captureLogger(); - const r = logger.withRequest(); - r.emit(); - r.emit(); - await logger.flush(); - expect(captured).toHaveLength(1); - }); - - it("accepts a plain Error and a string via .error()", async () => { - const { logger, captured } = captureLogger(); - const r1 = logger.withRequest(); - r1.error(new Error("boom")).emit({ success: false, status_code: 500, kind: "http" }); - const r2 = logger.withRequest(); - r2.error("oops").emit({ success: false, status_code: 500, kind: "http" }); - await logger.flush(); - - expect(captured[0]?.error?.message).toBe("boom"); - expect(captured[1]?.error?.message).toBe("oops"); - expect(captured[0]?.error?.code).toBe("UNKNOWN"); - }); - - it("traceparent() round-trips the trace and span IDs", () => { - const { logger } = captureLogger(); - const r = logger.withRequest({ traceId: "c".repeat(32), spanId: "d".repeat(16) }); - expect(r.traceparent()).toBe(`00-${"c".repeat(32)}-${"d".repeat(16)}-01`); - }); - - it("propagates parentSpanId and parentRequestId into the emitted event", async () => { - const { logger, captured } = captureLogger(); - logger.withRequest({ parentSpanId: "f".repeat(16), parentRequestId: "req_parent" }).emit(); - await logger.flush(); - expect(captured[0]?.request.parent_span_id).toBe("f".repeat(16)); - expect(captured[0]?.parent_request_id).toBe("req_parent"); - }); -}); diff --git a/packages/waylog-ts/src/__tests__/parity.test.ts b/packages/waylog-ts/src/__tests__/parity.test.ts new file mode 100644 index 0000000..30265cc --- /dev/null +++ b/packages/waylog-ts/src/__tests__/parity.test.ts @@ -0,0 +1,124 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { begin, fail, finalize, finalizeAborted, finalizePanic, finalizeTimeout, from, init, newError, recordOutgoingSpan, setField, setHTTPStatus, stepSync, suppress } from "../index.js"; +import type { WideEvent } from "../types.js"; + +const fixtureDir = join(process.cwd(), "..", "..", "testdata", "fixtures", "v2"); + +function capture() { + const lines: string[] = []; + init({ service: "checkout", env: "test", output: (line) => lines.push(line) }); + return { + event(): WideEvent { + const raw = lines.find((line) => line.trim().startsWith("{")); + if (!raw) throw new Error("no event"); + return JSON.parse(raw) as WideEvent; + }, + }; +} + +function fixture(name: string): WideEvent { + return JSON.parse(readFileSync(join(fixtureDir, name), "utf8")) as WideEvent; +} + +function masked(ev: WideEvent): unknown { + const v = JSON.parse(JSON.stringify(ev)) as any; + for (const k of ["event_id", "trace_id", "span_id", "parent_span_id", "ts_start", "ts_end"]) { + if (k in v) v[k] = v[k] === "" ? "__MASKED_EMPTY__" : "__MASKED__"; + } + v.duration_ms = 0; + for (const s of v.steps ?? []) { + if ("span_id" in s) s.span_id = s.span_id === "" ? "__MASKED_EMPTY__" : "__MASKED__"; + s.start_ms = 0; + s.duration_ms = 0; + } + for (const l of v.logs ?? []) l.ts_offset_ms = 0; + return normalize(v); +} + +function normalize(v: unknown): unknown { + if (Array.isArray(v)) return v.length === 0 ? undefined : v.map(normalize).filter((x) => x !== undefined); + if (v && typeof v === "object") { + const out: Record = {}; + for (const [k, val] of Object.entries(v)) { + const n = normalize(val); + if (n !== undefined && n !== "") out[k] = n; + } + return Object.keys(out).length === 0 ? undefined : out; + } + return v; +} + +function expectParity(got: WideEvent, name: string): void { + expect(masked(got)).toEqual(masked(fixture(name))); +} + +describe("TS fixture parity", () => { + it("ok-simple", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "1".repeat(32), spanId: "1".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout", status: 200 }); + stepSync(ctx, "db.load_cart", () => undefined); + await finalize(ctx); + expectParity(h.event(), "ok-simple.json"); + }); + + it("error-payment-cascade", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "2".repeat(32), spanId: "2".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout", status: 200 }); + setField(ctx, "user", { id: "u_123" }); + stepSync(ctx, "db.load_cart", () => undefined); + const err = newError("PMT_502", { reason: "upstream gateway 5xx" }); + try { + stepSync(ctx, "payment.charge", () => { + from(ctx).warn("retrying payment"); + recordOutgoingSpan(ctx, "3".repeat(16), "payment", "POST /charge"); + fail(ctx, err); + throw err; + }); + } catch { + // Expected by the fixture driver. + } + setHTTPStatus(ctx, 502); + await finalize(ctx); + expectParity(h.event(), "error-payment-cascade.json"); + }); + + it("error-panic", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "6".repeat(32), spanId: "7".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout", status: 500 }); + await finalizePanic(ctx); + expectParity(h.event(), "error-panic.json"); + }); + + it("suppressed-healthcheck", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "5".repeat(32), spanId: "6".repeat(16) }); + setField(ctx, "http", { method: "GET", route: "/healthz", status: 200 }); + suppress(ctx); + await finalize(ctx); + expectParity(h.event(), "suppressed-healthcheck.json"); + }); + + it("aborted-cancel", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "4".repeat(32), spanId: "5".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout" }); + await finalizeAborted(ctx); + expectParity(h.event(), "aborted-cancel.json"); + }); + + it("timeout-watchdog", async () => { + const h = capture(); + const ctx = begin({}, { traceId: "3".repeat(32), spanId: "4".repeat(16) }); + setField(ctx, "http", { method: "POST", route: "/checkout" }); + stepSync(ctx, "db.load_cart", () => undefined); + stepSync(ctx, "payment.charge", () => { + void finalizeTimeout(ctx); + }); + expectParity(h.event(), "timeout-watchdog.json"); + }); +}); diff --git a/packages/waylog-ts/src/__tests__/transport.test.ts b/packages/waylog-ts/src/__tests__/transport.test.ts index 41e4b8a..6956861 100644 --- a/packages/waylog-ts/src/__tests__/transport.test.ts +++ b/packages/waylog-ts/src/__tests__/transport.test.ts @@ -1,96 +1,107 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { Transport } from "../transport.js"; -import { SCHEMA_VERSION, type WideEvent } from "../types.js"; +import { describe, expect, it, vi } from "vitest"; +import { Transport, normalizeIngestUrl } from "../transport.js"; +import type { WideEvent } from "../types.js"; -function sampleEvent(n = 0): WideEvent { +function event(id: string, status: WideEvent["status"] = "ok"): WideEvent { return { - schema_version: SCHEMA_VERSION, - event_name: "svc.request", - timestamp: new Date(0).toISOString(), - user: { id: "u", tier: "free", region: "us", vip: false }, - request: { trace_id: "a".repeat(32), span_id: "b".repeat(16), flow: "f", feature_flags: [] }, - system: { service: "svc", version: "1", deployment_id: "", env: "test" }, - outcome: { success: true, status_code: 200 + n, kind: "http" }, - metrics: { latency_ms: n }, + schema_version: "2.0", + event_id: id, + ts_start: new Date(0).toISOString(), + ts_end: new Date(1).toISOString(), + duration_ms: 1, + kind: "http", + service: "checkout", + env: "test", + trace_id: "a".repeat(32), + span_id: "b".repeat(16), + parent_span_id: "", + status, }; } -describe("Transport", () => { - beforeEach(() => vi.useFakeTimers()); - afterEach(() => vi.useRealTimers()); +describe("v2 transport", () => { + it("normalizes ingest URLs", () => { + expect(normalizeIngestUrl("http://x")).toBe("http://x/v1/events"); + expect(normalizeIngestUrl("http://x/v1/events")).toBe("http://x/v1/events"); + expect(normalizeIngestUrl("http://x?token=abc")).toBe("http://x/v1/events?token=abc"); + }); - it("flushes once batchSize is reached", async () => { - const fakeFetch = vi.fn(async () => new Response("ok", { status: 200 })); + it("supports single-event JSON mode", async () => { + const calls: RequestInit[] = []; const t = new Transport({ - endpoint: "http://x", - service: "svc", - batchSize: 2, - flushIntervalMs: 10_000, - fetch: fakeFetch as unknown as typeof fetch, + service: "checkout", + env: "test", + ingestUrl: "http://x", + batchMode: false, + fetch: vi.fn(async (_url, init) => { + calls.push(init!); + return new Response("ok", { status: 200 }); + }) as unknown as typeof fetch, }); - t.enqueue(sampleEvent(1)); - expect(fakeFetch).not.toHaveBeenCalled(); - t.enqueue(sampleEvent(2)); - // The second enqueue triggers flush synchronously as a microtask. - await vi.waitFor(() => expect(fakeFetch).toHaveBeenCalledTimes(1)); - const call = fakeFetch.mock.calls[0]!; - const body = JSON.parse(call[1]!.body as string) as WideEvent[]; - expect(body).toHaveLength(2); + t.submit(event("json-1")); + await t.shutdown(); + expect(calls[0]?.headers).toMatchObject({ "Content-Type": "application/json" }); + expect(JSON.parse(String(calls[0]?.body))).toMatchObject({ event_id: "json-1" }); }); - it("drops events once queueMax is hit and reports count", () => { + it("posts NDJSON with auth", async () => { + const calls: RequestInit[] = []; const t = new Transport({ - endpoint: "http://x", - service: "svc", - queueMax: 1, - batchSize: 100, - flushIntervalMs: 10_000, - fetch: vi.fn() as unknown as typeof fetch, + service: "checkout", + env: "test", + ingestUrl: "http://x", + apiKey: "k", + fetch: vi.fn(async (_url, init) => { + calls.push(init!); + return new Response("ok", { status: 200 }); + }) as unknown as typeof fetch, }); - expect(t.enqueue(sampleEvent())).toBe(true); - expect(t.enqueue(sampleEvent())).toBe(false); - expect(t.droppedCount()).toBe(1); + t.submit(event("e1")); + await t.shutdown(); + expect(calls[0]?.headers).toMatchObject({ "Content-Type": "application/x-ndjson", Authorization: "Bearer k" }); + expect(String(calls[0]?.body)).toContain("\"event_id\":\"e1\""); }); - it("counts network failures as drops", async () => { + it("retries transient failures", async () => { + let calls = 0; const t = new Transport({ - endpoint: "http://x", - service: "svc", - batchSize: 1, + service: "checkout", + env: "test", + ingestUrl: "http://x", fetch: vi.fn(async () => { - throw new Error("econnrefused"); + calls++; + return new Response("x", { status: calls === 1 ? 500 : 200 }); }) as unknown as typeof fetch, }); - t.enqueue(sampleEvent()); - await vi.waitFor(() => expect(t.droppedCount()).toBe(1)); + t.submit(event("e1", "error")); + await t.shutdown(1000); + expect(calls).toBeGreaterThanOrEqual(2); + expect(t.failureCount()).toBe(1); }); - it("sends a single object (not an array) when batch size is 1", async () => { - const fakeFetch = vi.fn(async () => new Response("ok")); + it("counts permanent drops", async () => { const t = new Transport({ - endpoint: "http://x", - service: "svc", - batchSize: 1, - fetch: fakeFetch as unknown as typeof fetch, + service: "checkout", + env: "test", + ingestUrl: "http://x", + fetch: vi.fn(async () => new Response("bad", { status: 400 })) as unknown as typeof fetch, }); - t.enqueue(sampleEvent()); - await vi.waitFor(() => expect(fakeFetch).toHaveBeenCalled()); - const body = JSON.parse(fakeFetch.mock.calls[0]![1]!.body as string); - expect(Array.isArray(body)).toBe(false); - expect(body.event_name).toBe("svc.request"); + t.submit(event("bad", "error")); + await t.shutdown(); + expect(t.droppedCount()).toBe(1); }); - it("flushes remaining queue on close", async () => { - const fakeFetch = vi.fn(async () => new Response("ok")); + it("drops ok queue under pressure before priority", () => { const t = new Transport({ - endpoint: "http://x", - service: "svc", - batchSize: 100, - flushIntervalMs: 10_000, - fetch: fakeFetch as unknown as typeof fetch, + service: "checkout", + env: "test", + ingestUrl: "http://x", + maxInFlightBytes: 500, + fetch: vi.fn() as unknown as typeof fetch, }); - t.enqueue(sampleEvent()); - await t.close(); - expect(fakeFetch).toHaveBeenCalledTimes(1); + expect(t.submit(event("ok-1"))).toBe(true); + expect(t.submit(event("ok-2"))).toBe(true); + expect(t.submit(event("prio", "error"))).toBe(true); + expect(t.droppedCount()).toBeGreaterThan(0); }); }); diff --git a/packages/waylog-ts/src/error.ts b/packages/waylog-ts/src/error.ts index 424fac0..0a1645a 100644 --- a/packages/waylog-ts/src/error.ts +++ b/packages/waylog-ts/src/error.ts @@ -1,26 +1,17 @@ -import type { ErrorContext } from "./types.js"; +import { WaylogError, type ErrorOpts } from "./types.js"; -// createError builds a structured ErrorContext with the four schema 1.1 fields. -// Callers pass what they know; `code` and `message` are required. -export function createError(input: { - code: string; - message: string; - path?: string; - reason?: string; -}): ErrorContext { - if (!input.code) throw new Error("createError: code is required"); - if (!input.message) throw new Error("createError: message is required"); - const out: ErrorContext = { code: input.code, message: input.message }; - if (input.path) out.path = input.path; - if (input.reason) out.reason = input.reason; - return out; +const reservedCodes = new Set(["WAYLOG_TIMEOUT", "WAYLOG_ABORTED", "WAYLOG_PANIC", "WAYLOG_PARTIAL"]); + +export function newError(code: string, opts: ErrorOpts = {}): WaylogError { + if (!code) throw new Error("waylog: error code is required"); + if (reservedCodes.has(code)) throw new Error(`waylog: reserved error code ${code}`); + return new WaylogError(code, opts); +} + +export function isWaylogError(err: unknown): err is WaylogError { + return err instanceof WaylogError || (typeof err === "object" && err !== null && typeof (err as WaylogError).code === "string"); } -export function isErrorContext(x: unknown): x is ErrorContext { - return ( - typeof x === "object" && - x !== null && - typeof (x as ErrorContext).code === "string" && - typeof (x as ErrorContext).message === "string" - ); +export function isReservedCode(code: string): boolean { + return reservedCodes.has(code); } diff --git a/packages/waylog-ts/src/express.ts b/packages/waylog-ts/src/express.ts index 9ac02de..4ac0221 100644 --- a/packages/waylog-ts/src/express.ts +++ b/packages/waylog-ts/src/express.ts @@ -1,7 +1,23 @@ -import { createLogger, startRequest, type Logger } from "./logger.js"; -import type { RequestLogger, WaylogConfig } from "./types.js"; +import { + begin, + fail, + finalize, + finalizeAborted, + finalizeTimeout, + formatTraceparent, + from, + init, + newError, + parseTraceparent, + runWithContext, + setField, + setHTTPStatus, + spanId, + traceId, + type Context, +} from "./index.js"; +import type { Logger, WaylogConfig } from "./types.js"; -// Express-shaped types without taking a runtime dependency on express. type Req = { method?: string; route?: { path?: string }; @@ -16,43 +32,63 @@ type Res = { }; type Next = (err?: unknown) => void; -const LOGGER_KEY = Symbol.for("waylog.request.logger"); +const CTX_KEY = Symbol.for("waylog.v2.express.context"); -export interface ExpressMiddlewareOptions { - logger?: Logger; -} - -// waylog() creates (or reuses) a Logger, attaches a per-request RequestLogger -// to req[Symbol.for("waylog.request.logger")], and emits on response finish. -export function waylog(config: WaylogConfig & ExpressMiddlewareOptions) { - const logger = config.logger ?? createLogger(config); - - return function middleware(req: Req, res: Res, next: Next): void { - const { log, traceparent } = startRequest(logger, { - method: req.method, - route: req.route?.path ?? req.url, - traceparentHeader: pickHeader(req.headers["traceparent"]), - }); - (req as any)[LOGGER_KEY] = log; - res.setHeader?.("traceparent", traceparent); +export function middleware(config: WaylogConfig) { + init(config); + return function waylogExpress(req: Req, res: Res, next: Next): void { + const inbound = parseTraceparent(pickHeader(req.headers.traceparent)); + const ctx = begin(req as Context, { traceId: inbound?.traceId, parentSpanId: inbound?.spanId }); + setField(ctx, "http", { method: req.method ?? "", route: req.route?.path ?? req.url ?? "", status: 200 }); + (req as Record)[CTX_KEY] = ctx; + res.setHeader?.("traceparent", formatTraceparent(traceId(ctx), spanId(ctx))); - const finalize = (): void => { - if (log.emitted()) return; - log.emit({ success: res.statusCode < 500, status_code: res.statusCode }); + let finalized = false; + let timer: ReturnType | undefined; + const markFinalized = (): boolean => { + if (finalized) return false; + finalized = true; + if (timer) clearTimeout(timer); + return true; }; - res.on("finish", finalize); - res.on("close", finalize); - next(); + const finish = (): void => { + if (!markFinalized()) return; + setHTTPStatus(ctx, res.statusCode || 200); + if (res.statusCode >= 500) fail(ctx, newError(`HTTP_${res.statusCode}`, { reason: `HTTP ${res.statusCode}` })); + void finalize(ctx); + }; + const close = (): void => { + if (!markFinalized()) return; + void finalizeAborted(ctx); + }; + res.on("finish", finish); + res.on("close", close); + if (config.maxRequestAgeMs && config.maxRequestAgeMs > 0) { + timer = setTimeout(() => { + if (!markFinalized()) return; + void finalizeTimeout(ctx); + }, config.maxRequestAgeMs); + } + try { + runWithContext(ctx, () => next()); + } catch (err) { + markFinalized(); + setHTTPStatus(ctx, 500); + fail(ctx, newError("ERR", { reason: err instanceof Error ? err.message : String(err) })); + void finalize(ctx); + throw err; + } }; } -export function useLogger(req: Req): RequestLogger { - const l = (req as any)[LOGGER_KEY] as RequestLogger | undefined; - if (!l) throw new Error("waylog: useLogger() called before waylog() middleware was registered"); - return l; +export const waylog = middleware; + +export function useLogger(req: Req): Logger { + const ctx = (req as Record)[CTX_KEY]; + if (!ctx) throw new Error("waylog: useLogger() called before middleware was registered"); + return from(ctx); } function pickHeader(v: string | string[] | undefined): string | undefined { - if (Array.isArray(v)) return v[0]; - return v; + return Array.isArray(v) ? v[0] : v; } diff --git a/packages/waylog-ts/src/hono.ts b/packages/waylog-ts/src/hono.ts index 5285474..87e405c 100644 --- a/packages/waylog-ts/src/hono.ts +++ b/packages/waylog-ts/src/hono.ts @@ -1,54 +1,64 @@ -import { createLogger, startRequest, type Logger } from "./logger.js"; -import type { RequestLogger, WaylogConfig } from "./types.js"; +import { + begin, + fail, + finalize, + finalizeTimeout, + formatTraceparent, + from, + init, + newError, + parseTraceparent, + runWithContext, + setField, + setHTTPStatus, + spanId, + traceId, + type Context, +} from "./index.js"; +import type { Logger, WaylogConfig } from "./types.js"; -// Hono-shaped context without runtime dependency on hono. type Ctx = { req: { method: string; routePath?: string; path?: string; header: (n: string) => string | undefined }; - res: { headers: { set: (n: string, v: string) => void } }; + res: { headers: { set: (n: string, v: string) => void }; status?: number }; set: (key: string, value: unknown) => void; get: (key: string) => unknown; }; -const LOGGER_KEY = "__waylog_request_logger__"; +const CTX_KEY = "__waylog_v2_context__"; -export interface HonoMiddlewareOptions { - logger?: Logger; -} - -// Hono-style middleware: returns an `async (c, next) => { ... }` function. -export function waylog(config: WaylogConfig & HonoMiddlewareOptions) { - const logger = config.logger ?? createLogger(config); - - return async function middleware(c: Ctx, next: () => Promise): Promise { - const { log, traceparent } = startRequest(logger, { - method: c.req.method, - route: c.req.routePath ?? c.req.path, - traceparentHeader: c.req.header("traceparent"), - }); - c.set(LOGGER_KEY, log); - c.res.headers.set("traceparent", traceparent); +export function middleware(config: WaylogConfig) { + init(config); + return async function waylogHono(c: Ctx, next: () => Promise): Promise { + const inbound = parseTraceparent(c.req.header("traceparent")); + const ctx = begin({}, { traceId: inbound?.traceId, parentSpanId: inbound?.spanId }); + setField(ctx, "http", { method: c.req.method, route: c.req.routePath ?? c.req.path ?? "", status: 200 }); + c.set(CTX_KEY, ctx); + c.res.headers.set("traceparent", formatTraceparent(traceId(ctx), spanId(ctx))); - let statusCode = 200; + let timer: ReturnType | undefined; + if (config.maxRequestAgeMs && config.maxRequestAgeMs > 0) { + timer = setTimeout(() => void finalizeTimeout(ctx), config.maxRequestAgeMs); + } try { - await next(); - // Hono surfaces final status through c.res, which middleware can read - // only after next() resolves. We peek defensively via any. - const anyRes = c.res as unknown as { status?: number }; - if (typeof anyRes.status === "number") statusCode = anyRes.status; + await runWithContext(ctx, () => next()); + setHTTPStatus(ctx, c.res.status ?? 200); + if ((c.res.status ?? 200) >= 500) fail(ctx, newError(`HTTP_${c.res.status ?? 500}`, { reason: `HTTP ${c.res.status ?? 500}` })); + await finalize(ctx); } catch (err) { - statusCode = 500; - log.error(err instanceof Error ? err : String(err)); + setHTTPStatus(ctx, 500); + fail(ctx, newError("ERR", { reason: err instanceof Error ? err.message : String(err) })); + await finalize(ctx); throw err; } finally { - if (!log.emitted()) { - log.emit({ success: statusCode < 500, status_code: statusCode }); - } + if (timer) clearTimeout(timer); } }; } -export function useLogger(c: Ctx): RequestLogger { - const l = c.get(LOGGER_KEY) as RequestLogger | undefined; - if (!l) throw new Error("waylog: useLogger() called before waylog() middleware was registered"); - return l; +export const waylog = middleware; + +export function useLogger(c: Ctx): Logger { + const ctx = c.get(CTX_KEY) as Context | undefined; + if (!ctx) throw new Error("waylog: useLogger() called before middleware was registered"); + return from(ctx); } diff --git a/packages/waylog-ts/src/index.ts b/packages/waylog-ts/src/index.ts index 3b0261d..a53c7f0 100644 --- a/packages/waylog-ts/src/index.ts +++ b/packages/waylog-ts/src/index.ts @@ -1,18 +1,48 @@ -export { createLogger, newTraceId, newSpanId, parseTraceparent } from "./logger.js"; -export type { Logger } from "./logger.js"; -export { createError, isErrorContext } from "./error.js"; +export { + begin, + explain, + fail, + finalize, + finalizeAborted, + finalizePanic, + finalizeTimeout, + formatTraceparent, + from, + init, + newError, + newSpanId, + newTraceId, + parseTraceparent, + recordOutgoingSpan, + runWithContext, + setField, + setHTTPRoute, + setHTTPStatus, + shutdown, + spanId, + stats, + step, + stepSync, + suppress, + traceId, +} from "./logger.js"; +export { Transport, normalizeIngestUrl } from "./transport.js"; +export { SCHEMA_VERSION, WaylogError } from "./types.js"; export type { - ErrorContext, - LoggerOptions, - MetricsContext, - OutcomeContext, - RequestContext, - RequestLogger, - RetryContext, - SetFields, - SystemContext, - UserContext, + Anchor, + BeginOptions, + Context, + Downstream, + ErrorOpts, + ErrorRef, + ExplainResult, + Fields, + Log, + Logger, + Stats, + Status, + Step, + StepError, WaylogConfig, WideEvent, } from "./types.js"; -export { SCHEMA_VERSION } from "./types.js"; diff --git a/packages/waylog-ts/src/logger.ts b/packages/waylog-ts/src/logger.ts index 8e0c3f4..899e63f 100644 --- a/packages/waylog-ts/src/logger.ts +++ b/packages/waylog-ts/src/logger.ts @@ -1,165 +1,616 @@ -import { isErrorContext } from "./error.js"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomBytes, randomUUID } from "node:crypto"; +import { isReservedCode, isWaylogError, newError as buildError } from "./error.js"; import { Transport } from "./transport.js"; import { SCHEMA_VERSION, - type ErrorContext, - type LoggerOptions, - type OutcomeContext, - type RequestLogger, - type SetFields, + WaylogError, + type Anchor, + type BeginOptions, + type Context, + type Downstream, + type ErrorOpts, + type ErrorRef, + type ExplainResult, + type Fields, + type Log, + type LogLevel, + type Logger, + type Stats, + type Status, + type Step, + type StepError, type WaylogConfig, type WideEvent, } from "./types.js"; -// 32/16-hex random IDs. W3C traceparent version is fixed at "00". -function randHex(bytes: number): string { - const buf = new Uint8Array(bytes); - crypto.getRandomValues(buf); - return Array.from(buf, (b) => b.toString(16).padStart(2, "0")).join(""); +const defaultMaxSteps = 128; +const defaultMaxLogs = 256; +const defaultMaxBufferBytes = 512 * 1024; +const ctxKey = Symbol.for("waylog.v2.request"); +const als = new AsyncLocalStorage(); + +type ActiveStep = { + name: string; + startedAt: number; + startMs: number; + spanId?: string; + downstream?: Downstream; +}; + +type RequestState = { + sdk: SDK; + eventId: string; + tsStart: Date; + traceId: string; + spanId: string; + parentSpanId: string; + fields: Fields; + steps: Step[]; + logs: Log[]; + errors: ErrorRef[]; + activeSteps: ActiveStep[]; + sealed: boolean; + suppressed: boolean; + headerOnly: boolean; + finalStatus?: Status; + anchor?: Anchor; + bufBytes: number; +}; + +class SDK { + readonly cfg: RequiredConfig; + readonly transport?: Transport; + readonly active = new Set(); + readonly devEnabled: boolean; + rateSecond = 0; + rateCount = 0; + stats: Stats = { + activeRequests: 0, + eventsEmitted: 0, + eventsSuppressed: 0, + stepsDropped: 0, + logsDropped: 0, + bytesDroppedFromBuffer: 0, + bufferOverflows: 0, + reservedCodeRejections: 0, + suppressedThenFailed: 0, + lateCompletionAfterEmit: 0, + eventsDropped: 0, + deliveryFailures: 0, + }; + + constructor(cfg: RequiredConfig) { + this.cfg = cfg; + this.devEnabled = cfg.devMode || cfg.env.toLowerCase() === "dev"; + if (cfg.ingestUrl) this.transport = new Transport(cfg); + } +} + +type RequiredConfig = WaylogConfig & { + output: NodeJS.WritableStream | ((line: string) => void); + maxSteps: number; + maxLogs: number; + maxBufferBytes: number; + maxInFlightBytes: number; + maxEventsPerSec: number; +}; + +let sdk: SDK | undefined; + +export function init(cfg: WaylogConfig): void { + if (!cfg.service) throw new Error("waylog: service is required"); + if (!cfg.env) throw new Error("waylog: env is required"); + sdk = new SDK({ + ...cfg, + output: cfg.output ?? process.stderr, + maxSteps: cfg.maxSteps && cfg.maxSteps > 0 ? cfg.maxSteps : defaultMaxSteps, + maxLogs: cfg.maxLogs && cfg.maxLogs > 0 ? cfg.maxLogs : defaultMaxLogs, + maxBufferBytes: cfg.maxBufferBytes && cfg.maxBufferBytes > 0 ? cfg.maxBufferBytes : defaultMaxBufferBytes, + maxInFlightBytes: cfg.maxInFlightBytes && cfg.maxInFlightBytes > 0 ? cfg.maxInFlightBytes : 10 << 20, + maxEventsPerSec: cfg.maxEventsPerSec ?? 0, + }); } -export function newTraceId(): string { return randHex(16); } -export function newSpanId(): string { return randHex(8); } +export async function shutdown(timeoutMs = 0): Promise { + if (!sdk) return; + await sdk.transport?.shutdown(timeoutMs); +} + +export function stats(): Stats { + if (!sdk) return emptyStats(); + return { + ...sdk.stats, + activeRequests: sdk.active.size, + eventsDropped: sdk.stats.eventsDropped + (sdk.transport?.droppedCount() ?? 0), + deliveryFailures: sdk.stats.deliveryFailures + (sdk.transport?.failureCount() ?? 0), + }; +} + +export function begin(ctx: Context = {}, opts: BeginOptions = {}): Context { + const s = ensureSDK(); + const now = opts.now ?? new Date(); + const r: RequestState = { + sdk: s, + eventId: randomUUID(), + tsStart: now, + traceId: opts.traceId || newTraceId(), + spanId: opts.spanId || newSpanId(), + parentSpanId: opts.parentSpanId ?? "", + fields: {}, + steps: [], + logs: [], + errors: [], + activeSteps: [], + sealed: false, + suppressed: false, + headerOnly: false, + bufBytes: 0, + }; + s.active.add(r); + return attach(ctx, r); +} + +export function runWithContext(ctx: Context, fn: () => T): T { + const r = requestFrom(ctx); + if (!r) return fn(); + return als.run(r, fn); +} + +export function from(ctx?: Context): Logger { + const r = ctx ? requestFrom(ctx) : als.getStore(); + return { + info: (msg, fields) => log(r, "info", msg, fields), + warn: (msg, fields) => log(r, "warn", msg, fields), + error: (msg, err, fields) => { + failState(r, err); + log(r, "error", msg, fields); + }, + }; +} + +export async function step(ctx: Context, name: string, fn: (ctx: Context) => Promise): Promise { + const r = requestFrom(ctx); + if (!r || !name) return fn(ctx); + pushStep(r, name); + try { + const v = await als.run(r, () => fn(ctx)); + closeStep(r, name); + return v; + } catch (err) { + closeStep(r, name, err); + throw err; + } +} + +export function stepSync(ctx: Context, name: string, fn: (ctx: Context) => T): T { + const r = requestFrom(ctx); + if (!r || !name) return fn(ctx); + pushStep(r, name); + try { + const v = als.run(r, () => fn(ctx)); + closeStep(r, name); + return v; + } catch (err) { + closeStep(r, name, err); + throw err; + } +} + +export function fail(ctx: Context, err: WaylogError): void { + failState(requestFrom(ctx), err); +} + +export function newError(code: string, opts: ErrorOpts = {}): WaylogError { + try { + return buildError(code, opts); + } catch (err) { + if (isReservedCode(code) && sdk) sdk.stats.reservedCodeRejections++; + throw err; + } +} + +export function suppress(ctx: Context): void { + const r = requestFrom(ctx); + if (!r || r.sealed || r.suppressed) return; + r.suppressed = true; + r.steps = []; + r.logs = []; + r.errors = []; + r.activeSteps = []; + r.anchor = undefined; + r.finalStatus = undefined; + r.headerOnly = false; + r.bufBytes = 0; +} + +export function setField(ctx: Context, key: string, value: unknown): void { + const r = requestFrom(ctx); + if (!r || !key || r.sealed || r.suppressed) return; + r.fields[key] = cloneDeep(value); +} + +export function setHTTPStatus(ctx: Context, status: number): void { + const r = requestFrom(ctx); + if (!r || r.sealed || r.suppressed) return; + const http = ensureHTTPFields(r); + http.status = status; +} + +export function setHTTPRoute(ctx: Context, route: string): void { + const r = requestFrom(ctx); + if (!r || !route || r.sealed || r.suppressed) return; + const http = ensureHTTPFields(r); + http.route = route; +} + +export function recordOutgoingSpan(ctx: Context, spanId: string, service: string, endpoint: string): void { + const r = requestFrom(ctx); + if (!r || r.sealed || r.suppressed || !spanId || r.activeSteps.length === 0) return; + const top = r.activeSteps[r.activeSteps.length - 1]!; + top.spanId = spanId; + top.downstream = { service, endpoint, kind: "rpc" }; +} + +export async function finalize(ctx: Context): Promise { + return finalizeWith(ctx, "normal"); +} + +export async function finalizePanic(ctx: Context): Promise { + return finalizeWith(ctx, "panic"); +} + +export async function finalizeAborted(ctx: Context): Promise { + return finalizeWith(ctx, "aborted"); +} + +export async function finalizeTimeout(ctx: Context): Promise { + return finalizeWith(ctx, "timeout"); +} + +export async function explain(ctx: Context): Promise { + const r = requestFrom(ctx); + if (!r) throw new Error("waylog: no active request"); + const status = statusOf(r); + const http = (r.fields.http ?? {}) as Fields; + const downstream = r.steps.flatMap((s) => (s.downstream ? [s.downstream] : [])); + const trace = r.traceId; + const service = r.sdk.cfg.service; + const anchor = r.anchor; + return { + traceId: trace, + service, + route: String(http.route ?? ""), + status, + anchor, + path: [...r.steps], + logs: [...r.logs], + downstream, + toString() { + const anchorText = anchor ? `${anchor.step} ${anchor.error_code}` : "none"; + return `trace ${trace} service=${service} status=${status} anchor=${anchorText}`; + }, + }; +} + +export function newTraceId(): string { + return randomBytes(16).toString("hex"); +} + +export function newSpanId(): string { + return randomBytes(8).toString("hex"); +} -// parseTraceparent extracts the trace-id and parent-id from a W3C -// `00---` header. Returns undefined on malformed input so -// callers fall back to generating fresh IDs. export function parseTraceparent(h: string | undefined | null): { traceId: string; spanId: string } | undefined { if (!h) return undefined; const parts = h.trim().split("-"); if (parts.length !== 4) return undefined; const [version, traceId, spanId] = parts; if (version !== "00") return undefined; - if (!/^[0-9a-f]{32}$/.test(traceId!) || !/^[0-9a-f]{16}$/.test(spanId!)) return undefined; + if (!/^[0-9a-f]{32}$/.test(traceId ?? "") || !/^[0-9a-f]{16}$/.test(spanId ?? "")) return undefined; return { traceId: traceId!, spanId: spanId! }; } -export interface Logger { - withRequest(opts?: LoggerOptions): RequestLogger; - flush(): Promise; - close(): Promise; - droppedCount(): number; -} - -// startRequest is the shared setup middlewares run for every incoming request: -// parse inbound traceparent, create a RequestLogger, and return both the logger -// and the outbound traceparent header so the framework can set it however it -// prefers (Express res.setHeader vs Hono c.res.headers.set). -export function startRequest( - logger: Logger, - ctx: { method?: string; route?: string; traceparentHeader?: string }, -): { log: RequestLogger; traceparent: string } { - const tp = parseTraceparent(ctx.traceparentHeader); - const log = logger.withRequest({ - traceId: tp?.traceId, - parentSpanId: tp?.spanId, - httpMethod: ctx.method, - routeTemplate: ctx.route, - }); - return { log, traceparent: log.traceparent() }; -} - -class RequestLoggerImpl implements RequestLogger { - private event: WideEvent; - private wasEmitted = false; - private readonly startedAt: number; - private readonly transport: Transport; - - constructor(base: WideEvent, transport: Transport, startedAt: number) { - this.event = base; - this.transport = transport; - this.startedAt = startedAt; - } - - set(fields: SetFields): RequestLogger { - const e = this.event; - if (fields.user) e.user = { ...e.user, ...fields.user }; - if (fields.request) e.request = { ...e.request, ...fields.request }; - if (fields.system) e.system = { ...e.system, ...fields.system }; - if (fields.outcome) e.outcome = { ...e.outcome, ...fields.outcome }; - if (fields.metrics) e.metrics = { ...e.metrics, ...fields.metrics }; - if (fields.metadata) e.metadata = { ...(e.metadata ?? {}), ...fields.metadata }; - if (fields.error !== undefined) e.error = fields.error; - if (fields.retry !== undefined) e.retry = fields.retry; - if (fields.event_name !== undefined) e.event_name = fields.event_name; - if (fields.timestamp !== undefined) e.timestamp = fields.timestamp; - if (fields.parent_request_id !== undefined) e.parent_request_id = fields.parent_request_id; - return this; - } - - error(err: ErrorContext | Error | string): RequestLogger { - if (typeof err === "string") { - this.event.error = { code: "UNKNOWN", message: err }; - } else if (isErrorContext(err)) { - this.event.error = err; - } else { - this.event.error = { code: "UNKNOWN", message: err.message || "error" }; +export function formatTraceparent(traceId: string, spanId: string): string { + return `00-${traceId}-${spanId}-01`; +} + +export function traceId(ctx: Context): string { + return requestFrom(ctx)?.traceId ?? ""; +} + +export function spanId(ctx: Context): string { + return requestFrom(ctx)?.spanId ?? ""; +} + +function finalizeWith(ctx: Context, lifecycle: "normal" | "panic" | "aborted" | "timeout"): WideEvent | undefined { + const r = requestFrom(ctx); + if (!r) throw new Error("waylog: no active request"); + if (r.sealed) { + r.sdk.stats.lateCompletionAfterEmit++; + return undefined; + } + applyLifecycle(r, lifecycle); + r.sealed = true; + const ev = assemble(r, new Date()); + r.sdk.active.delete(r); + deliver(r.sdk, ev); + emitDevFinal(r.sdk, ev); + return ev; +} + +function applyLifecycle(r: RequestState, lifecycle: "normal" | "panic" | "aborted" | "timeout"): void { + if (r.suppressed) return; + const now = Date.now(); + if (lifecycle === "panic") { + markLifecycle(r, "error", "WAYLOG_PANIC"); + recordError(r, "WAYLOG_PANIC", "runtime panic recovered"); + flushActiveSteps(r, now, "error", { code: "WAYLOG_PANIC", reason: "runtime panic recovered" }); + } else if (lifecycle === "timeout") { + markLifecycle(r, "timeout", "WAYLOG_TIMEOUT"); + flushActiveSteps(r, now, "ok"); + } else if (lifecycle === "aborted") { + if (!r.anchor) markLifecycle(r, "aborted", "WAYLOG_ABORTED"); + flushActiveSteps(r, now, "ok"); + } +} + +function assemble(r: RequestState, tsEnd: Date): WideEvent { + const status = statusOf(r); + const fields = Object.keys(r.fields).length > 0 ? (r.sdk.cfg.redactor ? r.sdk.cfg.redactor(r.fields) : r.fields) : undefined; + const ev: WideEvent = { + schema_version: SCHEMA_VERSION, + event_id: r.eventId, + ts_start: r.tsStart.toISOString(), + ts_end: tsEnd.toISOString(), + duration_ms: Math.max(0, tsEnd.getTime() - r.tsStart.getTime()), + kind: "http", + service: r.sdk.cfg.service, + env: r.sdk.cfg.env, + ...(r.sdk.cfg.version ? { version: r.sdk.cfg.version } : {}), + trace_id: r.traceId, + span_id: r.spanId, + parent_span_id: r.parentSpanId, + status, + ...(fields ? { fields } : {}), + }; + if (status === "suppressed") return ev; + if (r.anchor) ev.anchor = r.anchor; + if (!r.headerOnly) { + if (r.steps.length > 0) ev.steps = r.steps; + if (r.logs.length > 0) ev.logs = r.logs; + if (r.errors.length > 0) ev.errors = r.errors; + } + return ev; +} + +function deliver(s: SDK, ev: WideEvent): void { + if (!allowEvent(s, ev)) { + s.stats.eventsDropped++; + return; + } + if (s.transport) { + if (s.transport.submit(ev)) { + s.stats.eventsEmitted++; + if (ev.status === "suppressed") s.stats.eventsSuppressed++; } - return this; - } - - emit(outcome?: Partial): void { - if (this.wasEmitted) return; - this.wasEmitted = true; - if (outcome) this.event.outcome = { ...this.event.outcome, ...outcome }; - this.event.metrics.latency_ms = Math.max(0, Date.now() - this.startedAt); - // event_name default: ".request" on success, ".error" on failure. - if (!this.event.event_name) { - const suffix = this.event.outcome.success ? "request" : "error"; - this.event.event_name = `${this.event.system.service}.${suffix}`; + return; + } + writeOutput(s.cfg.output, `${JSON.stringify(ev)}\n`); + s.stats.eventsEmitted++; + if (ev.status === "suppressed") s.stats.eventsSuppressed++; +} + +function allowEvent(s: SDK, ev: WideEvent): boolean { + if (s.cfg.maxEventsPerSec <= 0 || ev.status === "error" || ev.status === "timeout" || ev.status === "partial" || ev.status === "aborted") return true; + const now = Math.floor(Date.now() / 1000); + if (s.rateSecond !== now) { + s.rateSecond = now; + s.rateCount = 0; + } + if (s.rateCount >= s.cfg.maxEventsPerSec) return false; + s.rateCount++; + return true; +} + +function log(r: RequestState | undefined, level: LogLevel, msg: string, fields?: Fields): void { + if (!r || r.sealed || r.suppressed || r.headerOnly) return; + const entry: Log = { ts_offset_ms: Date.now() - r.tsStart.getTime(), level, msg, ...(fields ? { fields: cloneTop(fields) } : {}) }; + const bytes = logBytes(entry); + if (r.logs.length >= r.sdk.cfg.maxLogs || r.bufBytes + bytes > r.sdk.cfg.maxBufferBytes) { + r.sdk.stats.logsDropped++; + r.sdk.stats.bytesDroppedFromBuffer += bytes; + return; + } + r.logs.push(entry); + r.bufBytes += bytes; + emitDevLog(r.sdk, level, msg, fields); +} + +function pushStep(r: RequestState, name: string): void { + const now = Date.now(); + r.activeSteps.push({ name, startedAt: now, startMs: now - r.tsStart.getTime() }); +} + +function closeStep(r: RequestState, name: string, err?: unknown): void { + const active = popStep(r, name); + if (r.sealed || r.suppressed) return; + const stepErr = errorFromUnknown(err); + const step: Step = { + name, + ...(active.spanId ? { span_id: active.spanId } : {}), + start_ms: active.startMs, + duration_ms: Math.max(0, Date.now() - active.startedAt), + status: stepErr ? "error" : "ok", + ...(active.downstream ? { downstream: active.downstream } : {}), + ...(stepErr ? { error: stepErr } : {}), + }; + if (stepErr) { + recordError(r, stepErr.code || "ERR", stepErr.reason); + if (!r.anchor) r.anchor = { step: name, error_code: stepErr.code || "ERR" }; + } + addStep(r, step); +} + +function addStep(r: RequestState, step: Step): void { + if (r.headerOnly) { + r.sdk.stats.stepsDropped++; + return; + } + const bytes = stepBytes(step); + if (r.steps.length >= r.sdk.cfg.maxSteps || r.bufBytes + bytes > r.sdk.cfg.maxBufferBytes) { + r.sdk.stats.stepsDropped++; + r.sdk.stats.bytesDroppedFromBuffer += bytes; + return; + } + r.steps.push(step); + r.bufBytes += bytes; +} + +function flushActiveSteps(r: RequestState, now: number, status: "ok" | "error", err?: StepError): void { + for (const active of r.activeSteps) { + addStep(r, { + name: active.name, + ...(active.spanId ? { span_id: active.spanId } : {}), + start_ms: active.startMs, + duration_ms: Math.max(0, now - active.startedAt), + status, + ...(active.downstream ? { downstream: active.downstream } : {}), + ...(err ? { error: err } : {}), + }); + } + r.activeSteps = []; +} + +function failState(r: RequestState | undefined, err: WaylogError): void { + if (!r || !err || r.sealed) return; + if (r.suppressed) { + r.sdk.stats.suppressedThenFailed++; + return; + } + if (isReservedCode(err.code)) { + r.sdk.stats.reservedCodeRejections++; + return; + } + recordError(r, err.code, err.reason); + if (!r.anchor) r.anchor = { step: r.activeSteps.at(-1)?.name ?? "request", error_code: err.code }; +} + +function markLifecycle(r: RequestState, status: Status, code: string): void { + r.finalStatus = status; + r.anchor = { step: r.activeSteps.at(-1)?.name ?? "request", error_code: code }; +} + +function statusOf(r: RequestState): Status { + if (r.suppressed) return "suppressed"; + if (r.finalStatus) return r.finalStatus; + if (r.anchor) return "error"; + return "ok"; +} + +function recordError(r: RequestState, code: string, reason?: string): void { + if (!code || r.errors.some((e) => e.code === code)) return; + r.errors.push({ code, ...(reason ? { reason } : {}) }); +} + +function errorFromUnknown(err: unknown): StepError | undefined { + if (err == null) return undefined; + if (isWaylogError(err)) { + if (isReservedCode(err.code) && sdk) { + sdk.stats.reservedCodeRejections++; + return { code: "ERR", reason: err.message }; + } + return { code: err.code, ...(err.reason ? { reason: err.reason } : {}) }; + } + return { code: "ERR", reason: err instanceof Error ? err.message : String(err) }; +} + +function popStep(r: RequestState, name: string): ActiveStep { + let i = -1; + for (let n = r.activeSteps.length - 1; n >= 0; n--) { + if (r.activeSteps[n]?.name === name) { + i = n; + break; } - this.transport.enqueue(this.event); } + if (i < 0) return { name, startedAt: Date.now(), startMs: 0 }; + const [active] = r.activeSteps.splice(i, 1); + return active!; +} + +function attach(ctx: Context, r: RequestState): Context { + Object.defineProperty(ctx, ctxKey, { value: r, enumerable: false, configurable: true }); + return ctx; +} - emitted(): boolean { return this.wasEmitted; } +function requestFrom(ctx: Context | undefined): RequestState | undefined { + if (!ctx) return als.getStore(); + return (ctx as Record)[ctxKey] ?? als.getStore(); +} + +function ensureSDK(): SDK { + if (!sdk) throw new Error("waylog: init() must be called first"); + return sdk; +} + +function ensureHTTPFields(r: RequestState): Fields { + const existing = r.fields.http; + if (existing && typeof existing === "object" && !Array.isArray(existing)) return existing as Fields; + const next: Fields = {}; + r.fields.http = next; + return next; +} + +function writeOutput(output: NodeJS.WritableStream | ((line: string) => void), line: string): void { + if (typeof output === "function") output(line); + else output.write(line); +} + +function emitDevLog(s: SDK, level: LogLevel, msg: string, fields?: Fields): void { + if (!s.devEnabled) return; + writeOutput(s.cfg.output, `[${level.toUpperCase()}] ${msg}${fields ? ` ${JSON.stringify(fields)}` : ""}\n`); +} - traceparent(): string { - return `00-${this.event.request.trace_id}-${this.event.request.span_id ?? newSpanId()}-01`; +function emitDevFinal(s: SDK, ev: WideEvent): void { + if (!s.devEnabled) return; + writeOutput(s.cfg.output, `${JSON.stringify(ev, null, 2)}\n`); +} + +function cloneTop(fields: Fields): Fields { + return { ...fields }; +} + +function cloneDeep(v: unknown): unknown { + if (Array.isArray(v)) return v.map(cloneDeep); + if (v && typeof v === "object") { + const out: Fields = {}; + for (const [k, val] of Object.entries(v)) out[k] = cloneDeep(val); + return out; } + return v; } -export function createLogger(config: WaylogConfig): Logger { - if (!config.service) throw new Error("createLogger: service is required"); - const transport = new Transport(config); - const now = config.now ?? (() => new Date()); +function stepBytes(step: Step): number { + return JSON.stringify(step).length; +} + +function logBytes(entry: Log): number { + return JSON.stringify(entry).length; +} +function emptyStats(): Stats { return { - withRequest(opts: LoggerOptions = {}): RequestLogger { - const traceId = opts.traceId ?? newTraceId(); - const spanId = opts.spanId ?? newSpanId(); - const base: WideEvent = { - schema_version: SCHEMA_VERSION, - event_name: "", - timestamp: now().toISOString(), - user: { - id: opts.user?.id ?? "", - tier: opts.user?.tier ?? "", - region: opts.user?.region ?? "", - vip: opts.user?.vip ?? false, - }, - request: { - trace_id: traceId, - span_id: spanId, - parent_span_id: opts.parentSpanId, - flow: opts.flow ?? "", - feature_flags: [], - http_method: opts.httpMethod, - route_template: opts.routeTemplate, - }, - system: { - service: config.service, - version: config.version ?? "", - deployment_id: "", - env: config.env ?? "", - }, - outcome: { success: true, status_code: 200, kind: "http" }, - metrics: { latency_ms: 0 }, - parent_request_id: opts.parentRequestId, - }; - return new RequestLoggerImpl(base, transport, Date.now()); - }, - flush: () => transport.flush(), - close: () => transport.close(), - droppedCount: () => transport.droppedCount(), + activeRequests: 0, + eventsEmitted: 0, + eventsSuppressed: 0, + stepsDropped: 0, + logsDropped: 0, + bytesDroppedFromBuffer: 0, + bufferOverflows: 0, + reservedCodeRejections: 0, + suppressedThenFailed: 0, + lateCompletionAfterEmit: 0, + eventsDropped: 0, + deliveryFailures: 0, }; } diff --git a/packages/waylog-ts/src/nest.ts b/packages/waylog-ts/src/nest.ts new file mode 100644 index 0000000..f5f7156 --- /dev/null +++ b/packages/waylog-ts/src/nest.ts @@ -0,0 +1,20 @@ +import { middleware as expressMiddleware, useLogger } from "./express.js"; + +export { useLogger }; + +export function middleware(config: Parameters[0]) { + return expressMiddleware(config); +} + +export function interceptor(config: Parameters[0]) { + const mw = expressMiddleware(config); + return { + intercept(context: { switchToHttp(): { getRequest(): unknown; getResponse(): unknown } }, next: { handle(): { subscribe(observer: { error(err: unknown): void; complete(): void }): void } }) { + const http = context.switchToHttp(); + mw(http.getRequest() as any, http.getResponse() as any, (err?: unknown) => { + if (err) throw err; + }); + return next.handle(); + }, + }; +} diff --git a/packages/waylog-ts/src/next.ts b/packages/waylog-ts/src/next.ts new file mode 100644 index 0000000..5eb0d32 --- /dev/null +++ b/packages/waylog-ts/src/next.ts @@ -0,0 +1,40 @@ +import { begin, fail, finalize, finalizeTimeout, from, init, newError, parseTraceparent, runWithContext, setField, setHTTPStatus, type Context } from "./index.js"; +import type { Logger, WaylogConfig } from "./types.js"; + +type NextRequestLike = { + method: string; + url: string; + headers: { get(name: string): string | null }; +}; +type NextResponseLike = { status?: number }; + +export type NextHandler = (req: NextRequestLike, ctx: Context) => Promise | T; + +export function middleware(config: WaylogConfig, handler: NextHandler) { + init(config); + return async function waylogNext(req: NextRequestLike): Promise { + const inbound = parseTraceparent(req.headers.get("traceparent")); + const ctx = begin({}, { traceId: inbound?.traceId, parentSpanId: inbound?.spanId }); + setField(ctx, "http", { method: req.method, route: new URL(req.url).pathname, status: 200 }); + let timer: ReturnType | undefined; + if (config.maxRequestAgeMs && config.maxRequestAgeMs > 0) timer = setTimeout(() => void finalizeTimeout(ctx), config.maxRequestAgeMs); + try { + const res = await runWithContext(ctx, () => handler(req, ctx)); + setHTTPStatus(ctx, res.status ?? 200); + if ((res.status ?? 200) >= 500) fail(ctx, newError(`HTTP_${res.status ?? 500}`, { reason: `HTTP ${res.status ?? 500}` })); + await finalize(ctx); + return res; + } catch (err) { + setHTTPStatus(ctx, 500); + fail(ctx, newError("ERR", { reason: err instanceof Error ? err.message : String(err) })); + await finalize(ctx); + throw err; + } finally { + if (timer) clearTimeout(timer); + } + }; +} + +export function useLogger(ctx: Context): Logger { + return from(ctx); +} diff --git a/packages/waylog-ts/src/transport.ts b/packages/waylog-ts/src/transport.ts index 1b56e37..99bcd77 100644 --- a/packages/waylog-ts/src/transport.ts +++ b/packages/waylog-ts/src/transport.ts @@ -1,46 +1,80 @@ -import type { WaylogConfig, WideEvent } from "./types.js"; +import type { Status, WaylogConfig, WideEvent } from "./types.js"; + +const defaultMaxBatch = 256; +const defaultMaxBatchSize = 1 << 20; +const defaultBatchAgeMs = 50; +const defaultInFlightCap = 10 << 20; +const defaultOkBudgetPct = 70; +const defaultMaxRetries = 5; + +type Queued = { + ev: WideEvent; + bytes: number; + attempts: number; +}; + +type DeliveryResult = { + success: boolean; + retryable: boolean; + retryAfterMs: number; +}; -// Transport batches WideEvents and POSTs them to /v1/events. Bounded queue, -// non-blocking enqueue (returns false on overflow), drop counter exposed so -// callers can surface loss. export class Transport { - private readonly endpoint: string; + private readonly url: string; private readonly apiKey?: string; private readonly fetchImpl: typeof fetch; - private readonly batchSize: number; - private readonly flushIntervalMs: number; - private readonly queueMax: number; + private readonly maxBatch: number; + private readonly maxBatchSize: number; + private readonly batchAgeMs: number; + private readonly inFlightCap: number; + private readonly okBudgetPct: number; + private readonly maxRetries: number; + private readonly batchMode: boolean; - private queue: WideEvent[] = []; - private dropped = 0; - private timer: ReturnType | null = null; + private okQ: Queued[] = []; + private priorityQ: Queued[] = []; + private okBytes = 0; + private priorityBytes = 0; + private timer: ReturnType | undefined; + private flushing = false; private closed = false; + private dropped = 0; + private failures = 0; + private jsonPosts = new Set>(); constructor(config: WaylogConfig) { - if (!config.endpoint) throw new Error("Transport: endpoint is required"); - this.endpoint = config.endpoint.replace(/\/+$/, "") + "/v1/events"; + if (!config.ingestUrl) throw new Error("Transport: ingestUrl is required"); + this.url = normalizeIngestUrl(config.ingestUrl); this.apiKey = config.apiKey; this.fetchImpl = config.fetch ?? fetch; - this.batchSize = config.batchSize ?? 32; - this.flushIntervalMs = config.flushIntervalMs ?? 1000; - this.queueMax = config.queueMax ?? 1024; + this.maxBatch = defaultMaxBatch; + this.maxBatchSize = defaultMaxBatchSize; + this.batchAgeMs = defaultBatchAgeMs; + this.inFlightCap = config.maxInFlightBytes && config.maxInFlightBytes > 0 ? config.maxInFlightBytes : defaultInFlightCap; + this.okBudgetPct = defaultOkBudgetPct; + this.maxRetries = defaultMaxRetries; + this.batchMode = config.batchMode ?? true; } - enqueue(ev: WideEvent): boolean { + submit(ev: WideEvent): boolean { if (this.closed) { this.dropped++; return false; } - if (this.queue.length >= this.queueMax) { + if (!this.batchMode) { + const post = this.postJSON(ev); + this.jsonPosts.add(post); + void post.finally(() => this.jsonPosts.delete(post)); + return true; + } + const item: Queued = { ev, bytes: estimateEventSize(ev), attempts: 0 }; + const accepted = isPriorityStatus(ev.status) ? this.enqueuePriority(item) : this.enqueueOK(item); + if (!accepted) { this.dropped++; return false; } - this.queue.push(ev); - if (this.queue.length >= this.batchSize) { - void this.flush(); - } else if (!this.timer) { - this.timer = setTimeout(() => void this.flush(), this.flushIntervalMs); - } + if (this.ready()) void this.flush(false); + else this.armTimer(); return true; } @@ -48,36 +82,229 @@ export class Transport { return this.dropped; } - queueLength(): number { - return this.queue.length; + failureCount(): number { + return this.failures; + } + + async shutdown(timeoutMs = 0): Promise { + this.closed = true; + if (this.timer) clearTimeout(this.timer); + const drain = Promise.all([ + this.flush(true), + Promise.allSettled([...this.jsonPosts]).then(() => undefined), + ]).then(() => undefined); + if (timeoutMs <= 0) { + await drain; + return; + } + await Promise.race([drain, new Promise((resolve) => setTimeout(resolve, timeoutMs))]); + } + + private enqueueOK(item: Queued): boolean { + const okCap = Math.floor((this.inFlightCap * this.okBudgetPct) / 100); + if (item.bytes > okCap) return false; + while (this.okBytes + item.bytes > okCap && this.okQ.length > 0) this.dropOK(); + if (this.okBytes + item.bytes > okCap) return false; + this.okQ.push(item); + this.okBytes += item.bytes; + return true; } - async flush(): Promise { + private enqueuePriority(item: Queued): boolean { + if (item.bytes > this.inFlightCap) return false; + while (this.totalBytes() + item.bytes > this.inFlightCap && this.okQ.length > 0) this.dropOK(); + while (this.totalBytes() + item.bytes > this.inFlightCap && this.priorityQ.length > 0) this.dropPriority(); + if (this.totalBytes() + item.bytes > this.inFlightCap) return false; + this.priorityQ.push(item); + this.priorityBytes += item.bytes; + return true; + } + + private ready(): boolean { + return this.priorityQ.length >= this.maxBatch || this.okQ.length >= this.maxBatch || this.priorityBytes >= this.maxBatchSize || this.okBytes >= this.maxBatchSize; + } + + private armTimer(): void { + if (this.timer) return; + this.timer = setTimeout(() => { + this.timer = undefined; + void this.flush(true); + }, this.batchAgeMs); + } + + private async flush(force: boolean): Promise { + if (this.flushing) return; + this.flushing = true; if (this.timer) { clearTimeout(this.timer); - this.timer = null; + this.timer = undefined; } - // Drain one batch per POST so a burst doesn't block the event loop on a - // single JSON.stringify of the whole queue. Network failures count as drops - // (no retry — caller sees the loss via droppedCount()). - while (this.queue.length > 0) { - const batch = this.queue.splice(0, this.batchSize); - const headers: Record = { "Content-Type": "application/json" }; - if (this.apiKey) headers["Authorization"] = `Bearer ${this.apiKey}`; - try { - await this.fetchImpl(this.endpoint, { - method: "POST", - headers, - body: JSON.stringify(batch.length === 1 ? batch[0] : batch), - }); - } catch { - this.dropped += batch.length; + try { + while (true) { + const batch = this.takeBatch(force); + if (batch.items.length === 0) break; + const result = await this.post(batch.items.map((item) => item.ev)); + if (!result.success && result.retryable) { + await this.retry(batch.items, batch.priority, result.retryAfterMs); + continue; + } } + } finally { + this.flushing = false; + if (!this.closed && (this.okQ.length > 0 || this.priorityQ.length > 0)) this.armTimer(); } } - async close(): Promise { - this.closed = true; - await this.flush(); + private takeBatch(force: boolean): { priority: boolean; items: Queued[] } { + const priority = this.takeFrom(this.priorityQ, force); + if (priority.length > 0) { + this.priorityBytes -= sumBytes(priority); + return { priority: true, items: priority }; + } + const ok = this.takeFrom(this.okQ, force); + if (ok.length > 0) { + this.okBytes -= sumBytes(ok); + return { priority: false, items: ok }; + } + return { priority: false, items: [] }; + } + + private takeFrom(q: Queued[], force: boolean): Queued[] { + const bytes = sumBytes(q); + if (q.length === 0 || (!force && q.length < this.maxBatch && bytes < this.maxBatchSize)) return []; + const out: Queued[] = []; + let batchBytes = 0; + while (q.length > 0 && out.length < this.maxBatch) { + const next = q[0]!; + if (out.length > 0 && batchBytes + next.bytes > this.maxBatchSize) break; + out.push(q.shift()!); + batchBytes += next.bytes; + } + return out; + } + + private async post(events: WideEvent[]): Promise { + const body = events.map((ev) => JSON.stringify(ev)).join("\n") + "\n"; + const headers: Record = { "Content-Type": "application/x-ndjson" }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + try { + const resp = await this.fetchImpl(this.url, { method: "POST", headers, body }); + if (resp.status >= 200 && resp.status < 300) return { success: true, retryable: false, retryAfterMs: 0 }; + if (resp.status === 429 || resp.status >= 500) { + this.failures += events.length; + return { success: false, retryable: true, retryAfterMs: retryAfterMs(resp.headers.get("Retry-After")) }; + } + this.dropped += events.length; + return { success: false, retryable: false, retryAfterMs: 0 }; + } catch { + this.failures += events.length; + return { success: false, retryable: true, retryAfterMs: 0 }; + } } + + private async retry(items: Queued[], priority: boolean, retryAfterMs: number): Promise { + const retry: Queued[] = []; + for (const item of items) { + item.attempts++; + if (item.attempts > this.maxRetries) this.dropped++; + else retry.push(item); + } + if (retry.length === 0) return; + await new Promise((resolve) => setTimeout(resolve, retryAfterMs || backoffMs(retry))); + this.requeueFront(retry, priority); + } + + private requeueFront(items: Queued[], priority: boolean): void { + const bytes = sumBytes(items); + if (priority) { + while (this.totalBytes() + bytes > this.inFlightCap && this.okQ.length > 0) this.dropOK(); + while (this.totalBytes() + bytes > this.inFlightCap && this.priorityQ.length > 0) this.dropPriorityTail(); + if (this.totalBytes() + bytes > this.inFlightCap) { + this.dropped += items.length; + return; + } + this.priorityQ.unshift(...items); + this.priorityBytes += bytes; + return; + } + const okCap = Math.floor((this.inFlightCap * this.okBudgetPct) / 100); + while (this.okBytes + bytes > okCap && this.okQ.length > 0) this.dropOK(); + if (this.okBytes + bytes > okCap) { + this.dropped += items.length; + return; + } + this.okQ.unshift(...items); + this.okBytes += bytes; + } + + private async postJSON(ev: WideEvent): Promise { + const headers: Record = { "Content-Type": "application/json" }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + try { + const resp = await this.fetchImpl(this.url, { method: "POST", headers, body: JSON.stringify(ev) }); + if (resp.status >= 200 && resp.status < 300) return; + if (resp.status === 429 || resp.status >= 500) this.failures++; + else this.dropped++; + } catch { + this.failures++; + } + } + + private totalBytes(): number { + return this.okBytes + this.priorityBytes; + } + + private dropOK(): void { + const item = this.okQ.shift(); + if (!item) return; + this.okBytes -= item.bytes; + this.dropped++; + } + + private dropPriority(): void { + const item = this.priorityQ.shift(); + if (!item) return; + this.priorityBytes -= item.bytes; + this.dropped++; + } + + private dropPriorityTail(): void { + const item = this.priorityQ.pop(); + if (!item) return; + this.priorityBytes -= item.bytes; + this.dropped++; + } +} + +export function normalizeIngestUrl(raw: string): string { + const url = new URL(raw); + if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`Transport: invalid ingestUrl ${raw}`); + const path = url.pathname.replace(/\/+$/, ""); + url.pathname = path.endsWith("/v1/events") ? path : `${path}/v1/events`; + return url.toString(); +} + +export function isPriorityStatus(status: Status): boolean { + return status === "error" || status === "timeout" || status === "partial" || status === "aborted"; +} + +function estimateEventSize(ev: WideEvent): number { + return JSON.stringify(ev).length + 1; +} + +function sumBytes(items: Queued[]): number { + return items.reduce((n, item) => n + item.bytes, 0); +} + +function retryAfterMs(raw: string | null): number { + if (!raw) return 0; + const seconds = Number.parseInt(raw, 10); + if (Number.isFinite(seconds)) return seconds * 1000; + const t = Date.parse(raw); + return Number.isFinite(t) ? Math.max(0, t - Date.now()) : 0; +} + +function backoffMs(items: Queued[]): number { + const attempt = Math.max(1, ...items.map((item) => item.attempts)); + return Math.min(5000, 100 * 2 ** (attempt - 1)); } diff --git a/packages/waylog-ts/src/types.ts b/packages/waylog-ts/src/types.ts index 84eb5d5..93ef98d 100644 --- a/packages/waylog-ts/src/types.ts +++ b/packages/waylog-ts/src/types.ts @@ -1,119 +1,149 @@ -// WideEvent shape mirrors pkg/event/event.go (schema 1.1). Fields that map to -// Go `omitempty` are optional here so the wire payload matches exactly. +export const SCHEMA_VERSION = "2.0"; -export const SCHEMA_VERSION = "1.1"; +export type Status = "ok" | "error" | "timeout" | "partial" | "aborted" | "suppressed"; +export type StepStatus = "ok" | "error"; +export type LogLevel = "info" | "warn" | "error"; +export type Fields = Record; -export interface UserContext { - id: string; - tier: string; - region: string; - vip: boolean; +export interface WaylogConfig { + service: string; + env: string; + version?: string; + output?: NodeJS.WritableStream | ((line: string) => void); + ingestUrl?: string; + apiKey?: string; + devMode?: boolean; + maxSteps?: number; + maxLogs?: number; + maxRequestAgeMs?: number; + maxBufferBytes?: number; + maxInFlightBytes?: number; + maxEventsPerSec?: number; + batchMode?: boolean; + redactor?: (fields: Fields) => Fields; + fetch?: typeof fetch; } -export interface RequestContext { - trace_id: string; - span_id?: string; - parent_span_id?: string; - http_method?: string; - route_template?: string; - flow: string; - feature_flags: string[]; - correlation_id?: string; - attempt?: number; - transport_kind?: string; +export interface Stats { + activeRequests: number; + eventsEmitted: number; + eventsSuppressed: number; + stepsDropped: number; + logsDropped: number; + bytesDroppedFromBuffer: number; + bufferOverflows: number; + reservedCodeRejections: number; + suppressedThenFailed: number; + lateCompletionAfterEmit: number; + eventsDropped: number; + deliveryFailures: number; } -export interface SystemContext { - service: string; - version: string; - deployment_id: string; - env: string; - downstream_service?: string; - caller_service?: string; +export interface Anchor { + step: string; + error_code: string; + kind?: string; } -export interface OutcomeContext { - success: boolean; - status_code: number; - kind: string; +export interface Downstream { + service?: string; + endpoint?: string; + kind?: string; } -export interface ErrorContext { - code: string; - path?: string; - message: string; +export interface StepError { + code?: string; reason?: string; + cause?: string; } -export interface RetryContext { - of?: number; - previous_attempt_id?: string; +export interface Step { + name: string; + span_id?: string; + start_ms: number; + duration_ms: number; + status: StepStatus; + downstream?: Downstream; + error?: StepError; } -export interface MetricsContext { - latency_ms: number; +export interface Log { + ts_offset_ms: number; + level: LogLevel; + msg: string; + fields?: Fields; } -export interface WideEvent { - schema_version: string; - event_name: string; - timestamp: string; - user: UserContext; - request: RequestContext; - system: SystemContext; - outcome: OutcomeContext; - error?: ErrorContext; - metrics: MetricsContext; - parent_request_id?: string; - metadata?: Record; - retry?: RetryContext; +export interface ErrorRef { + code: string; + reason?: string; } -export interface WaylogConfig { - endpoint: string; +export interface WideEvent { + schema_version: string; + event_id: string; + ts_start: string; + ts_end: string; + duration_ms: number; + kind: "http"; service: string; + env: string; version?: string; - env?: string; - apiKey?: string; - batchSize?: number; - flushIntervalMs?: number; - queueMax?: number; - fetch?: typeof fetch; - now?: () => Date; + trace_id: string; + span_id: string; + parent_span_id: string; + status: Status; + anchor?: Anchor; + steps?: Step[]; + logs?: Log[]; + fields?: Fields; + errors?: ErrorRef[]; +} + +export interface ErrorOpts { + reason?: string; + cause?: string; +} + +export class WaylogError extends Error { + readonly code: string; + readonly reason?: string; + readonly causeText?: string; + + constructor(code: string, opts: ErrorOpts = {}) { + super(opts.reason || code); + this.name = "WaylogError"; + this.code = code; + this.reason = opts.reason; + this.causeText = opts.cause; + } } -export interface LoggerOptions { +export interface Logger { + info(msg: string, fields?: Fields): void; + warn(msg: string, fields?: Fields): void; + error(msg: string, err: WaylogError, fields?: Fields): void; +} + +export interface Context { + readonly __waylogContext?: unique symbol; +} + +export interface BeginOptions { traceId?: string; spanId?: string; parentSpanId?: string; - flow?: string; - user?: Partial; - httpMethod?: string; - routeTemplate?: string; - parentRequestId?: string; -} - -// SetFields accepts partial sub-contexts so callers can patch one field at a -// time (e.g. `set({ user: { tier: "pro" } })`) without re-specifying the -// whole sub-context. -export interface SetFields { - user?: Partial; - request?: Partial; - system?: Partial; - outcome?: Partial; - metrics?: Partial; - metadata?: Record; - error?: ErrorContext; - retry?: RetryContext; - event_name?: string; - timestamp?: string; - parent_request_id?: string; + now?: Date; } -export interface RequestLogger { - set(fields: SetFields): RequestLogger; - error(err: ErrorContext | Error | string): RequestLogger; - emit(outcome?: Partial): void; - emitted(): boolean; - traceparent(): string; +export interface ExplainResult { + traceId: string; + service: string; + route: string; + status: Status; + anchor?: Anchor; + path: Step[]; + logs: Log[]; + downstream: Downstream[]; + toString(): string; } From dd2dea96deff1320fd7dc1a4fac6f30690f3ec84 Mon Sep 17 00:00:00 2001 From: skota-hash Date: Mon, 27 Apr 2026 02:53:36 -0400 Subject: [PATCH 8/8] fix(sdk): phase 1 sdk lifecycle and delivery hardening - preserve existing failure anchors when panic, timeout, or abort lifecycle finalizers run - keep panic-in-step middleware anchoring on the panicking step while still closing the active step before re-panicking - parse /v1/events ingest response envelopes in Go and TypeScript transports - expose rejected event counts through Go StatsSnapshot and TypeScript stats() - shut down the previous Go delivery transport when Init replaces a drained SDK state - align TypeScript reserved WAYLOG_* error handling with Go by returning undefined and counting the rejection - tighten Go and TypeScript parity normalization so empty identity strings remain meaningful - add regression tests for lifecycle anchor preservation, envelope rejection stats, re-init shutdown, step panic cleanup, reserved-code handling, and parity strictness - update README and SDK examples to reflect the Phase 1 v2 SDK surface - unignore v2 plan and SDK example docs so Phase 1 closeout docs can be tracked Verification: - npm run build - npm test - go test ./pkg/event/v2/... ./pkg/transport/http/... ./pkg/waylog/v2/... ./pkg/waylog/http/... ./pkg/waylog/chi/... ./pkg/waylog/gin/... ./pkg/waylog/echo/... - go test -race ./pkg/waylog/v2 ./pkg/waylog/http ./pkg/transport/http - make ci - make bench-gate --- .gitignore | 3 +- README.md | 13 +- docs/sdk-examples.md | 183 ++++++++++++++++++ packages/waylog-ts/src/__tests__/core.test.ts | 34 +++- .../waylog-ts/src/__tests__/parity.test.ts | 4 +- .../waylog-ts/src/__tests__/transport.test.ts | 15 ++ packages/waylog-ts/src/error.ts | 4 +- packages/waylog-ts/src/logger.ts | 49 +++-- packages/waylog-ts/src/transport.ts | 33 +++- packages/waylog-ts/src/types.ts | 3 +- pkg/transport/http/batch.go | 28 ++- pkg/transport/http/client.go | 20 +- pkg/transport/http/client_test.go | 39 ++++ pkg/waylog/v2/assemble.go | 14 +- pkg/waylog/v2/parity/runner.go | 7 +- pkg/waylog/v2/request.go | 12 +- pkg/waylog/v2/sdk_test.go | 103 ++++++++++ pkg/waylog/v2/step.go | 28 ++- pkg/waylog/v2/waylog.go | 42 ++-- 19 files changed, 578 insertions(+), 56 deletions(-) create mode 100644 docs/sdk-examples.md diff --git a/.gitignore b/.gitignore index 4d0aaed..0dfc197 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,5 @@ profile.cov !docs/internals.md !docs/env.md -!docs/waylog-sdk-contract.md \ No newline at end of file +!docs/waylog-sdk-contract.md +!docs/sdk-examples.md diff --git a/README.md b/README.md index 7294f1c..bce4062 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,11 @@ app.post("/buy", (req, res) => { ```go import ( - waylog "github.com/sssmaran/WaylogCLI/pkg" - wayloghttp "github.com/sssmaran/WaylogCLI/pkg/http" + "context" + "net/http" + + waylog "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" ) func main() { @@ -86,11 +89,11 @@ func main() { }) defer waylog.Shutdown(context.Background()) - http.Handle("/", wayloghttp.Middleware(yourHandler)) + http.Handle("/", wayloghttp.HTTP(yourHandler)) } ``` -Schema 1.1 helpers (`WithErrorReason`, `WithErrorPath`, `WithParentRequestID`, `WithMetadataKey`, `WithAttempt`, `WithRetry`) are additive. +The recommended SDK path is framework middleware plus `waylog.From(ctx)` / `useLogger(...)` inside handlers. Low-level request APIs such as `Begin`, `Finalize`, and `setField` are for adapter authors, tests, and unusual custom integrations. Full copy-paste examples for `net/http`, chi, gin, echo, standalone TypeScript, Express, Hono, Next.js, and NestJS are in [`docs/sdk-examples.md`](docs/sdk-examples.md). ### OTLP/HTTP traces @@ -267,7 +270,7 @@ Public alpha. APIs may break before 1.0. **Shipped:** -- Go SDK (schema 1.1 accessors) and TypeScript SDK (`@waylog/sdk`, ESM, Node 18+, Express + Hono middleware) +- Go SDK v2 (`net/http`, chi, gin, echo) and TypeScript SDK v2 (`@waylog/sdk`, ESM, Node 18+, standalone core, Express, Hono, Next.js, NestJS) - OTLP/HTTP traces at `/v1/otlp/v1/traces` (Phase A — traces only) - durable ingest with WAL + replay - hot graph with flattened 3-node model + dedicated trace store diff --git a/docs/sdk-examples.md b/docs/sdk-examples.md new file mode 100644 index 0000000..76f353c --- /dev/null +++ b/docs/sdk-examples.md @@ -0,0 +1,183 @@ +# SDK Examples + +These are the recommended copy-paste integration shapes for Waylog v2. Prefer framework middleware plus a request-scoped logger. Use low-level `Begin` / `Finalize` APIs only when writing a custom adapter or a deterministic test/smoke driver. + +## Go `net/http` + +```go +package main + +import ( + "context" + "net/http" + + waylog "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" + wayloghttp "github.com/sssmaran/WaylogCLI/pkg/waylog/http" +) + +func main() { + _ = waylog.Init(waylog.Config{ + Service: "checkout", + Env: "prod", + IngestURL: "http://localhost:8080", + }) + defer waylog.Shutdown(context.Background()) + + mux := http.NewServeMux() + mux.Handle("/buy", wayloghttp.HTTP(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + waylog.From(r.Context()).Info("cart loaded", waylog.F{"cart_id": "c_123"}) + w.WriteHeader(http.StatusOK) + }))) + + _ = http.ListenAndServe(":3000", mux) +} +``` + +## Go chi + +```go +import ( + "net/http" + + "github.com/go-chi/chi/v5" + waylogchi "github.com/sssmaran/WaylogCLI/pkg/waylog/chi" + waylog "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +r := chi.NewRouter() +r.Use(waylogchi.Middleware) +r.Post("/buy/{id}", func(w http.ResponseWriter, r *http.Request) { + waylog.From(r.Context()).Info("checkout started") + w.WriteHeader(http.StatusOK) +}) +``` + +## Go gin + +```go +import ( + "net/http" + + "github.com/gin-gonic/gin" + wayloggin "github.com/sssmaran/WaylogCLI/pkg/waylog/gin" + waylog "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +r := gin.New() +r.Use(wayloggin.Middleware()) +r.POST("/buy/:id", func(c *gin.Context) { + waylog.From(c.Request.Context()).Info("checkout started") + c.Status(http.StatusOK) +}) +``` + +## Go echo + +```go +import ( + "net/http" + + "github.com/labstack/echo/v4" + waylogecho "github.com/sssmaran/WaylogCLI/pkg/waylog/echo" + waylog "github.com/sssmaran/WaylogCLI/pkg/waylog/v2" +) + +e := echo.New() +e.Use(waylogecho.Middleware()) +e.POST("/buy/:id", func(c echo.Context) error { + waylog.From(c.Request().Context()).Info("checkout started") + return c.NoContent(http.StatusOK) +}) +``` + +## TypeScript Standalone + +```ts +import { begin, finalize, from, init, setField, step } from "@waylog/sdk"; + +init({ + service: "worker", + env: "prod", + ingestUrl: "http://localhost:8080", + apiKey: process.env.WAYLOG_WRITE_KEY, +}); + +const ctx = begin({}); +setField(ctx, "http", { method: "JOB", route: "queue:purchase", status: 200 }); +await step(ctx, "payment.charge", async () => { + from(ctx).info("charging payment", { cart_id: "c_123" }); +}); +await finalize(ctx); +``` + +## TypeScript Express + +```ts +import { waylog, useLogger } from "@waylog/sdk/express"; + +app.use(waylog({ + service: "checkout", + env: "prod", + ingestUrl: "http://localhost:8080", + apiKey: process.env.WAYLOG_WRITE_KEY, +})); + +app.post("/buy/:id", (req, res) => { + useLogger(req).info("checkout started", { cart_id: req.params.id }); + res.sendStatus(200); +}); +``` + +## TypeScript Hono + +```ts +import { Hono } from "hono"; +import { waylog, useLogger } from "@waylog/sdk/hono"; + +const app = new Hono(); +app.use("*", waylog({ + service: "checkout", + env: "prod", + ingestUrl: "http://localhost:8080", +})); + +app.post("/buy/:id", (c) => { + useLogger(c).info("checkout started", { cart_id: c.req.param("id") }); + return c.text("ok"); +}); +``` + +## TypeScript Next.js + +```ts +import { middleware as withWaylog, useLogger } from "@waylog/sdk/next"; + +export const POST = withWaylog({ + service: "checkout", + env: "prod", + ingestUrl: "http://localhost:8080", +}, async (_req, ctx) => { + useLogger(ctx).info("checkout started"); + return Response.json({ ok: true }, { status: 200 }); +}); +``` + +## TypeScript NestJS + +```ts +import { MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; +import { middleware as waylog } from "@waylog/sdk/nest"; + +@Module({}) +export class AppModule implements NestModule { + configure(consumer: MiddlewareConsumer) { + consumer + .apply(waylog({ + service: "checkout", + env: "prod", + ingestUrl: "http://localhost:8080", + })) + .forRoutes("*"); + } +} +``` diff --git a/packages/waylog-ts/src/__tests__/core.test.ts b/packages/waylog-ts/src/__tests__/core.test.ts index f50d172..444c95a 100644 --- a/packages/waylog-ts/src/__tests__/core.test.ts +++ b/packages/waylog-ts/src/__tests__/core.test.ts @@ -52,7 +52,7 @@ describe("v2 core lifecycle", () => { it("records explicit failures and local explain output", async () => { harness(); const ctx = begin({}, { traceId: "2".repeat(32), spanId: "2".repeat(16) }); - const err = newError("PMT_502", { reason: "upstream gateway 5xx" }); + const err = newError("PMT_502", { reason: "upstream gateway 5xx" })!; expect(() => stepSync(ctx, "payment.charge", () => { fail(ctx, err); throw err; @@ -112,6 +112,38 @@ describe("v2 core lifecycle", () => { expect(ev.steps?.[0]?.downstream?.service).toBe("payment"); }); + it("panic and timeout preserve explicit failure anchors", async () => { + const panicH = harness(); + const panicCtx = begin({}); + fail(panicCtx, newError("PMT_502", { reason: "payment failed first" })); + await finalizePanic(panicCtx); + expect(panicH.event().anchor?.error_code).toBe("PMT_502"); + expect(panicH.event().status).toBe("error"); + + const timeoutH = harness(); + const timeoutCtx = begin({}); + fail(timeoutCtx, newError("PMT_502", { reason: "payment failed first" })); + await finalizeTimeout(timeoutCtx); + expect(timeoutH.event().anchor?.error_code).toBe("PMT_502"); + expect(timeoutH.event().status).toBe("timeout"); + }); + + it("panic finalization replaces synthetic step panic anchors with WAYLOG_PANIC", async () => { + const h = harness(); + const ctx = begin({}); + expect(() => stepSync(ctx, "payment.charge", () => { + throw new Error("boom"); + })).toThrow("boom"); + await finalizePanic(ctx); + expect(h.event().anchor).toEqual({ step: "payment.charge", error_code: "WAYLOG_PANIC" }); + }); + + it("reserved error codes are rejected without throwing", () => { + harness(); + expect(newError("WAYLOG_TIMEOUT")).toBeUndefined(); + expect(stats().reservedCodeRejections).toBe(1); + }); + it("rate limiting drops ok before priority", async () => { const lines: string[] = []; init({ service: "checkout", env: "test", output: (line) => lines.push(line), maxEventsPerSec: 1 }); diff --git a/packages/waylog-ts/src/__tests__/parity.test.ts b/packages/waylog-ts/src/__tests__/parity.test.ts index 30265cc..bdea848 100644 --- a/packages/waylog-ts/src/__tests__/parity.test.ts +++ b/packages/waylog-ts/src/__tests__/parity.test.ts @@ -43,7 +43,7 @@ function normalize(v: unknown): unknown { const out: Record = {}; for (const [k, val] of Object.entries(v)) { const n = normalize(val); - if (n !== undefined && n !== "") out[k] = n; + if (n !== undefined) out[k] = n; } return Object.keys(out).length === 0 ? undefined : out; } @@ -70,7 +70,7 @@ describe("TS fixture parity", () => { setField(ctx, "http", { method: "POST", route: "/checkout", status: 200 }); setField(ctx, "user", { id: "u_123" }); stepSync(ctx, "db.load_cart", () => undefined); - const err = newError("PMT_502", { reason: "upstream gateway 5xx" }); + const err = newError("PMT_502", { reason: "upstream gateway 5xx" })!; try { stepSync(ctx, "payment.charge", () => { from(ctx).warn("retrying payment"); diff --git a/packages/waylog-ts/src/__tests__/transport.test.ts b/packages/waylog-ts/src/__tests__/transport.test.ts index 6956861..4457bf8 100644 --- a/packages/waylog-ts/src/__tests__/transport.test.ts +++ b/packages/waylog-ts/src/__tests__/transport.test.ts @@ -62,6 +62,21 @@ describe("v2 transport", () => { expect(String(calls[0]?.body)).toContain("\"event_id\":\"e1\""); }); + it("counts envelope rejections under 200 responses", async () => { + const t = new Transport({ + service: "checkout", + env: "test", + ingestUrl: "http://x", + fetch: vi.fn(async () => new Response( + JSON.stringify({ accepted: 0, duplicate: 0, rejected: [{ index: 0, event_id: "e1", reason: "validation_failed" }] }), + { status: 200, headers: { "Content-Type": "application/json" } }, + )) as unknown as typeof fetch, + }); + t.submit(event("e1", "error")); + await t.shutdown(); + expect(t.rejectedCount()).toBe(1); + }); + it("retries transient failures", async () => { let calls = 0; const t = new Transport({ diff --git a/packages/waylog-ts/src/error.ts b/packages/waylog-ts/src/error.ts index 0a1645a..08425e3 100644 --- a/packages/waylog-ts/src/error.ts +++ b/packages/waylog-ts/src/error.ts @@ -2,9 +2,9 @@ import { WaylogError, type ErrorOpts } from "./types.js"; const reservedCodes = new Set(["WAYLOG_TIMEOUT", "WAYLOG_ABORTED", "WAYLOG_PANIC", "WAYLOG_PARTIAL"]); -export function newError(code: string, opts: ErrorOpts = {}): WaylogError { +export function newError(code: string, opts: ErrorOpts = {}): WaylogError | undefined { if (!code) throw new Error("waylog: error code is required"); - if (reservedCodes.has(code)) throw new Error(`waylog: reserved error code ${code}`); + if (reservedCodes.has(code)) return undefined; return new WaylogError(code, opts); } diff --git a/packages/waylog-ts/src/logger.ts b/packages/waylog-ts/src/logger.ts index 899e63f..ad57062 100644 --- a/packages/waylog-ts/src/logger.ts +++ b/packages/waylog-ts/src/logger.ts @@ -55,6 +55,8 @@ type RequestState = { headerOnly: boolean; finalStatus?: Status; anchor?: Anchor; + anchorFromStepPanic: boolean; + panicStepHint?: string; bufBytes: number; }; @@ -78,6 +80,7 @@ class SDK { lateCompletionAfterEmit: 0, eventsDropped: 0, deliveryFailures: 0, + eventsRejected: 0, }; constructor(cfg: RequiredConfig) { @@ -124,6 +127,7 @@ export function stats(): Stats { activeRequests: sdk.active.size, eventsDropped: sdk.stats.eventsDropped + (sdk.transport?.droppedCount() ?? 0), deliveryFailures: sdk.stats.deliveryFailures + (sdk.transport?.failureCount() ?? 0), + eventsRejected: sdk.stats.eventsRejected + (sdk.transport?.rejectedCount() ?? 0), }; } @@ -145,6 +149,7 @@ export function begin(ctx: Context = {}, opts: BeginOptions = {}): Context { sealed: false, suppressed: false, headerOnly: false, + anchorFromStepPanic: false, bufBytes: 0, }; s.active.add(r); @@ -178,6 +183,7 @@ export async function step(ctx: Context, name: string, fn: (ctx: Context) => closeStep(r, name); return v; } catch (err) { + rememberPanicStep(r, name); closeStep(r, name, err); throw err; } @@ -192,22 +198,20 @@ export function stepSync(ctx: Context, name: string, fn: (ctx: Context) => T) closeStep(r, name); return v; } catch (err) { + rememberPanicStep(r, name); closeStep(r, name, err); throw err; } } -export function fail(ctx: Context, err: WaylogError): void { +export function fail(ctx: Context, err: WaylogError | undefined): void { failState(requestFrom(ctx), err); } -export function newError(code: string, opts: ErrorOpts = {}): WaylogError { - try { - return buildError(code, opts); - } catch (err) { - if (isReservedCode(code) && sdk) sdk.stats.reservedCodeRejections++; - throw err; - } +export function newError(code: string, opts: ErrorOpts = {}): WaylogError | undefined { + const err = buildError(code, opts); + if (!err && isReservedCode(code) && sdk) sdk.stats.reservedCodeRejections++; + return err; } export function suppress(ctx: Context): void { @@ -219,6 +223,8 @@ export function suppress(ctx: Context): void { r.errors = []; r.activeSteps = []; r.anchor = undefined; + r.anchorFromStepPanic = false; + r.panicStepHint = undefined; r.finalStatus = undefined; r.headerOnly = false; r.bufBytes = 0; @@ -343,11 +349,13 @@ function applyLifecycle(r: RequestState, lifecycle: "normal" | "panic" | "aborte if (r.suppressed) return; const now = Date.now(); if (lifecycle === "panic") { - markLifecycle(r, "error", "WAYLOG_PANIC"); + if (!r.anchor || r.anchorFromStepPanic) markLifecycle(r, "error", "WAYLOG_PANIC"); + else r.finalStatus = "error"; recordError(r, "WAYLOG_PANIC", "runtime panic recovered"); flushActiveSteps(r, now, "error", { code: "WAYLOG_PANIC", reason: "runtime panic recovered" }); } else if (lifecycle === "timeout") { - markLifecycle(r, "timeout", "WAYLOG_TIMEOUT"); + if (!r.anchor) markLifecycle(r, "timeout", "WAYLOG_TIMEOUT"); + else r.finalStatus = "timeout"; flushActiveSteps(r, now, "ok"); } else if (lifecycle === "aborted") { if (!r.anchor) markLifecycle(r, "aborted", "WAYLOG_ABORTED"); @@ -447,7 +455,10 @@ function closeStep(r: RequestState, name: string, err?: unknown): void { }; if (stepErr) { recordError(r, stepErr.code || "ERR", stepErr.reason); - if (!r.anchor) r.anchor = { step: name, error_code: stepErr.code || "ERR" }; + if (!r.anchor) { + r.anchor = { step: name, error_code: stepErr.code || "ERR" }; + r.anchorFromStepPanic = r.panicStepHint === name && (stepErr.code || "ERR") === "ERR"; + } } addStep(r, step); } @@ -482,7 +493,7 @@ function flushActiveSteps(r: RequestState, now: number, status: "ok" | "error", r.activeSteps = []; } -function failState(r: RequestState | undefined, err: WaylogError): void { +function failState(r: RequestState | undefined, err: WaylogError | undefined): void { if (!r || !err || r.sealed) return; if (r.suppressed) { r.sdk.stats.suppressedThenFailed++; @@ -493,12 +504,21 @@ function failState(r: RequestState | undefined, err: WaylogError): void { return; } recordError(r, err.code, err.reason); - if (!r.anchor) r.anchor = { step: r.activeSteps.at(-1)?.name ?? "request", error_code: err.code }; + if (!r.anchor) { + r.anchor = { step: r.activeSteps.at(-1)?.name ?? "request", error_code: err.code }; + r.anchorFromStepPanic = false; + } } function markLifecycle(r: RequestState, status: Status, code: string): void { r.finalStatus = status; - r.anchor = { step: r.activeSteps.at(-1)?.name ?? "request", error_code: code }; + const step = r.activeSteps.at(-1)?.name ?? (code === "WAYLOG_PANIC" && r.panicStepHint ? r.panicStepHint : "request"); + r.anchor = { step, error_code: code }; + r.anchorFromStepPanic = false; +} + +function rememberPanicStep(r: RequestState, name: string): void { + r.panicStepHint ??= name; } function statusOf(r: RequestState): Status { @@ -612,5 +632,6 @@ function emptyStats(): Stats { lateCompletionAfterEmit: 0, eventsDropped: 0, deliveryFailures: 0, + eventsRejected: 0, }; } diff --git a/packages/waylog-ts/src/transport.ts b/packages/waylog-ts/src/transport.ts index 99bcd77..746250f 100644 --- a/packages/waylog-ts/src/transport.ts +++ b/packages/waylog-ts/src/transport.ts @@ -19,6 +19,12 @@ type DeliveryResult = { retryAfterMs: number; }; +type IngestEnvelope = { + accepted?: number; + duplicate?: number; + rejected?: Array<{ index?: number; event_id?: string; reason?: string; detail?: string }>; +}; + export class Transport { private readonly url: string; private readonly apiKey?: string; @@ -40,6 +46,7 @@ export class Transport { private closed = false; private dropped = 0; private failures = 0; + private rejected = 0; private jsonPosts = new Set>(); constructor(config: WaylogConfig) { @@ -86,6 +93,10 @@ export class Transport { return this.failures; } + rejectedCount(): number { + return this.rejected; + } + async shutdown(timeoutMs = 0): Promise { this.closed = true; if (this.timer) clearTimeout(this.timer); @@ -189,7 +200,10 @@ export class Transport { if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; try { const resp = await this.fetchImpl(this.url, { method: "POST", headers, body }); - if (resp.status >= 200 && resp.status < 300) return { success: true, retryable: false, retryAfterMs: 0 }; + if (resp.status >= 200 && resp.status < 300) { + await this.recordEnvelope(resp); + return { success: true, retryable: false, retryAfterMs: 0 }; + } if (resp.status === 429 || resp.status >= 500) { this.failures += events.length; return { success: false, retryable: true, retryAfterMs: retryAfterMs(resp.headers.get("Retry-After")) }; @@ -242,7 +256,10 @@ export class Transport { if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; try { const resp = await this.fetchImpl(this.url, { method: "POST", headers, body: JSON.stringify(ev) }); - if (resp.status >= 200 && resp.status < 300) return; + if (resp.status >= 200 && resp.status < 300) { + await this.recordEnvelope(resp); + return; + } if (resp.status === 429 || resp.status >= 500) this.failures++; else this.dropped++; } catch { @@ -274,6 +291,18 @@ export class Transport { this.priorityBytes -= item.bytes; this.dropped++; } + + private async recordEnvelope(resp: Response): Promise { + const body = (await resp.text()).trim(); + if (!body) return; + let env: IngestEnvelope; + try { + env = JSON.parse(body) as IngestEnvelope; + } catch { + return; + } + this.rejected += env.rejected?.length ?? 0; + } } export function normalizeIngestUrl(raw: string): string { diff --git a/packages/waylog-ts/src/types.ts b/packages/waylog-ts/src/types.ts index 93ef98d..8ce666c 100644 --- a/packages/waylog-ts/src/types.ts +++ b/packages/waylog-ts/src/types.ts @@ -37,6 +37,7 @@ export interface Stats { lateCompletionAfterEmit: number; eventsDropped: number; deliveryFailures: number; + eventsRejected: number; } export interface Anchor { @@ -122,7 +123,7 @@ export class WaylogError extends Error { export interface Logger { info(msg: string, fields?: Fields): void; warn(msg: string, fields?: Fields): void; - error(msg: string, err: WaylogError, fields?: Fields): void; + error(msg: string, err: WaylogError | undefined, fields?: Fields): void; } export interface Context { diff --git a/pkg/transport/http/batch.go b/pkg/transport/http/batch.go index 2369ca0..32f5e3c 100644 --- a/pkg/transport/http/batch.go +++ b/pkg/transport/http/batch.go @@ -18,6 +18,19 @@ type deliveryResult struct { retryAfter time.Duration } +type ingestEnvelope struct { + Accepted int `json:"accepted"` + Duplicate int `json:"duplicate"` + Rejected []ingestRejected `json:"rejected"` +} + +type ingestRejected struct { + Index int `json:"index"` + EventID string `json:"event_id"` + Reason string `json:"reason"` + Detail string `json:"detail"` +} + func (c *Client) flushBatch(batch []*eventv2.Event) deliveryResult { if c.url == "" || len(batch) == 0 { return deliveryResult{success: true} @@ -49,11 +62,12 @@ func (c *Client) flushBatch(batch []*eventv2.Event) deliveryResult { c.recordFailure(len(batch)) return deliveryResult{retryable: true} } - _, _ = io.Copy(io.Discard, resp.Body) + respBody, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() switch { case resp.StatusCode >= 200 && resp.StatusCode < 300: + c.recordEnvelope(respBody) return deliveryResult{success: true} case isRetryableStatus(resp.StatusCode): c.recordFailure(len(batch)) @@ -67,6 +81,18 @@ func (c *Client) flushBatch(batch []*eventv2.Event) deliveryResult { } } +func (c *Client) recordEnvelope(body []byte) { + body = bytes.TrimSpace(body) + if len(body) == 0 { + return + } + var env ingestEnvelope + if err := json.Unmarshal(body, &env); err != nil { + return + } + c.recordRejected(len(env.Rejected)) +} + func retryAfter(raw string) time.Duration { raw = strings.TrimSpace(raw) if raw == "" { diff --git a/pkg/transport/http/client.go b/pkg/transport/http/client.go index 20adc28..af1f88f 100644 --- a/pkg/transport/http/client.go +++ b/pkg/transport/http/client.go @@ -44,6 +44,7 @@ type Client struct { dropped atomic.Int64 failures atomic.Int64 + rejected atomic.Int64 } func New(cfg Config) (*Client, error) { @@ -157,10 +158,12 @@ func (c *Client) submitSingle(ev *eventv2.Event) bool { c.recordFailure(1) return false } - _, _ = io.Copy(io.Discard, resp.Body) + respBody, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return true + before := c.Rejected() + c.recordEnvelope(respBody) + return c.Rejected() == before } if isRetryableStatus(resp.StatusCode) { c.recordFailure(1) @@ -184,6 +187,13 @@ func (c *Client) Failures() int64 { return c.failures.Load() } +func (c *Client) Rejected() int64 { + if c == nil { + return 0 + } + return c.rejected.Load() +} + func (c *Client) recordDrop(n int) { if n > 0 { c.dropped.Add(int64(n)) @@ -196,6 +206,12 @@ func (c *Client) recordFailure(n int) { } } +func (c *Client) recordRejected(n int) { + if n > 0 { + c.rejected.Add(int64(n)) + } +} + func isRetryableStatus(code int) bool { return code == http.StatusTooManyRequests || code >= 500 } diff --git a/pkg/transport/http/client_test.go b/pkg/transport/http/client_test.go index 6e74c0e..4bbbce9 100644 --- a/pkg/transport/http/client_test.go +++ b/pkg/transport/http/client_test.go @@ -61,6 +61,26 @@ func TestClientSinglePost(t *testing.T) { } } +func TestSinglePostCountsEnvelopeRejections(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":0,"duplicate":0,"rejected":[{"index":0,"event_id":"e1","reason":"validation_failed"}]}`) + })) + defer srv.Close() + + cli, err := New(Config{IngestURL: srv.URL}) + if err != nil { + t.Fatalf("New: %v", err) + } + if cli.Submit(validEvent("e1", eventv2.StatusError)) { + t.Fatal("single submit should report false when envelope rejects the event") + } + if got := cli.Rejected(); got != 1 { + t.Fatalf("Rejected=%d want 1", got) + } +} + func TestNDJSONBatchFlush(t *testing.T) { var batches atomic.Int64 var totalEvents atomic.Int64 @@ -101,6 +121,25 @@ func TestNDJSONBatchFlush(t *testing.T) { } } +func TestNDJSONBatchCountsEnvelopeRejections(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":0,"duplicate":0,"rejected":[{"index":0,"event_id":"e1","reason":"validation_failed"}]}`) + })) + defer srv.Close() + + cli, err := New(Config{IngestURL: srv.URL, BatchMode: true, BatchAgeMs: 1}) + if err != nil { + t.Fatalf("New: %v", err) + } + cli.Submit(validEvent("e1", eventv2.StatusError)) + cli.Shutdown(2 * time.Second) + if got := cli.Rejected(); got != 1 { + t.Fatalf("Rejected=%d want 1", got) + } +} + func TestQueueEvictsOKBeforePriority(t *testing.T) { var flushed []string q := newQueue(Config{ diff --git a/pkg/waylog/v2/assemble.go b/pkg/waylog/v2/assemble.go index b9fac2d..0564f3c 100644 --- a/pkg/waylog/v2/assemble.go +++ b/pkg/waylog/v2/assemble.go @@ -97,17 +97,25 @@ func (r *request) applyLifecycleLocked(lifecycle lifecycleKind) { // Panic is an app-observable failure, so it synthesizes both an // anchor and an entry in errors[] (§3.4: errors[] is the deduped // list of errors seen across the request). - r.markLifecycleLocked(eventv2.StatusError, eventv2.CodePanic) + if r.anchorStep == "" || r.anchorFromStepPanic { + r.markLifecycleLocked(eventv2.StatusError, eventv2.CodePanic) + } else { + r.finalStatus = eventv2.StatusError + } r.recordErrorLocked(eventv2.CodePanic, "runtime panic recovered") r.flushActiveStepsLocked(now) case lifecycleTimeout: // Watchdog expiry is a runtime decision, not an app error; the // lifecycle code lives only in anchor, not errors[]. - r.markLifecycleLocked(eventv2.StatusTimeout, eventv2.CodeTimeout) + if r.anchorStep == "" { + r.markLifecycleLocked(eventv2.StatusTimeout, eventv2.CodeTimeout) + } else { + r.finalStatus = eventv2.StatusTimeout + } r.flushActiveStepsLocked(now) case lifecycleAborted: // Cooperative cancel from the client; same errors[] rule as timeout. - // Preserve any explicit Fail anchor that was recorded before cancel. + // Preserve any failure anchor that was recorded before cancel. if r.anchorStep == "" { r.markLifecycleLocked(eventv2.StatusAborted, eventv2.CodeAborted) } diff --git a/pkg/waylog/v2/parity/runner.go b/pkg/waylog/v2/parity/runner.go index 5ab28bd..139cf50 100644 --- a/pkg/waylog/v2/parity/runner.go +++ b/pkg/waylog/v2/parity/runner.go @@ -203,9 +203,10 @@ func walkLogs(v any) any { return out } -// normalize collapses semantically-empty values (nil, "", [], {}) so that +// normalize collapses semantically-empty values (nil, [], {}) so that // an omitempty struct field marshalling away matches a fixture that -// explicitly included an empty array or empty object. +// explicitly included an empty array or empty object. Empty strings are kept +// because identity fields such as parent_span_id use "" as meaningful data. func normalize(v any) any { switch x := v.(type) { case map[string]any: @@ -239,8 +240,6 @@ func isSemanticEmpty(v any) bool { switch x := v.(type) { case nil: return true - case string: - return x == "" case map[string]any: return len(x) == 0 case []any: diff --git a/pkg/waylog/v2/request.go b/pkg/waylog/v2/request.go index 6596938..05aed6e 100644 --- a/pkg/waylog/v2/request.go +++ b/pkg/waylog/v2/request.go @@ -223,8 +223,10 @@ type request struct { // First observable failing step, or "request" sentinel for a Fail() with // no active step. Set once and never overwritten. - anchorStep string - anchorCode string + anchorStep string + anchorCode string + anchorFromStepPanic bool + panicStepHint string } type activeStep struct { @@ -266,6 +268,7 @@ func (r *request) addStepLocked(s stepBuf) { if r.anchorStep == "" { r.anchorStep = s.name r.anchorCode = s.err.Code + r.anchorFromStepPanic = r.panicStepHint == s.name && s.err.Code == "ERR" } } @@ -367,7 +370,12 @@ func (r *request) markLifecycleLocked(status eventv2.Status, code string) { } r.finalStatus = status r.anchorStep = r.activeStepLocked() + if code == eventv2.CodePanic && r.anchorStep == "request" && r.panicStepHint != "" { + r.anchorStep = r.panicStepHint + r.panicStepHint = "" + } r.anchorCode = code + r.anchorFromStepPanic = false } // degradeToHeaderOnlyLocked discards buffered detail and switches the request diff --git a/pkg/waylog/v2/sdk_test.go b/pkg/waylog/v2/sdk_test.go index e67141e..c52ce5b 100644 --- a/pkg/waylog/v2/sdk_test.go +++ b/pkg/waylog/v2/sdk_test.go @@ -5,10 +5,12 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "strings" "sync" + "sync/atomic" "testing" "time" @@ -109,6 +111,29 @@ func TestInitAllowedAfterDrain(t *testing.T) { } } +func TestInitAfterDrainShutsDownPreviousDelivery(t *testing.T) { + var delivered atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + delivered.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":1,"duplicate":0,"rejected":[]}`) + })) + defer srv.Close() + + newHarness(t, Config{IngestURL: srv.URL}) + ctx := Begin(context.Background(), BeginOptions{}) + if _, err := Finalize(ctx); err != nil { + t.Fatal(err) + } + + if err := Init(Config{Service: "next", Env: "test", Output: &bytes.Buffer{}}); err != nil { + t.Fatalf("re-init after drain: %v", err) + } + if got := delivered.Load(); got != 1 { + t.Fatalf("previous delivery flushes during re-init: got %d want 1", got) + } +} + func TestOKRequestEmitsValidEvent(t *testing.T) { h := newHarness(t, Config{}) ctx := Begin(context.Background(), BeginOptions{}) @@ -280,6 +305,61 @@ func TestFinalizeAbortedPreservesExistingExplicitFail(t *testing.T) { } } +func TestLifecyclePanicAndTimeoutPreserveExistingExplicitFailAnchor(t *testing.T) { + for _, tc := range []struct { + name string + finalize func(context.Context) (*eventv2.Event, error) + wantState eventv2.Status + }{ + {name: "panic", finalize: FinalizePanic, wantState: eventv2.StatusError}, + {name: "timeout", finalize: FinalizeTimeout, wantState: eventv2.StatusTimeout}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + Fail(ctx, NewError("PMT_502", WithReason("payment failed first"))) + + if _, err := tc.finalize(ctx); err != nil { + t.Fatalf("finalize: %v", err) + } + + h.validateLast() + ev := h.lastEvent() + if ev.Status != tc.wantState { + t.Fatalf("status=%s want %s", ev.Status, tc.wantState) + } + if ev.Anchor == nil || ev.Anchor.ErrorCode != "PMT_502" { + t.Fatalf("explicit failure anchor must survive lifecycle finalize: %+v", ev.Anchor) + } + }) + } +} + +func TestStepPanicClosesActiveStepBeforeRepanic(t *testing.T) { + h := newHarness(t, Config{}) + ctx := Begin(context.Background(), BeginOptions{}) + + func() { + defer func() { + if recover() == nil { + t.Fatal("expected panic") + } + }() + _ = StepVoid(ctx, "payment.charge", func(ctx context.Context) error { + panic("boom") + }) + }() + if _, err := FinalizePanic(ctx); err != nil { + t.Fatalf("FinalizePanic: %v", err) + } + + h.validateLast() + ev := h.lastEvent() + if len(ev.Steps) != 1 || ev.Steps[0].Name != "payment.charge" { + t.Fatalf("panic step was not closed into steps[]: %+v", ev.Steps) + } +} + func TestSetHTTPStatusUpdatesFieldsMap(t *testing.T) { h := newHarness(t, Config{}) ctx := Begin(context.Background(), BeginOptions{}) @@ -806,6 +886,29 @@ func TestMaxInFlightBytesDropsOversizedTransportEvent(t *testing.T) { } } +func TestIngestEnvelopeRejectionsSurfaceInStats(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"accepted":0,"duplicate":0,"rejected":[{"index":0,"event_id":"e1","reason":"validation_failed","detail":"bad"}]}`) + })) + defer srv.Close() + + newHarness(t, Config{IngestURL: srv.URL}) + ctx := Begin(context.Background(), BeginOptions{}) + if _, err := Finalize(ctx); err != nil { + t.Fatalf("Finalize: %v", err) + } + deadline, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := Shutdown(deadline); err != nil { + t.Fatalf("Shutdown: %v", err) + } + if got := Stats().EventsRejected; got != 1 { + t.Fatalf("EventsRejected=%d want 1", got) + } +} + func TestMaxEventsPerSecDropsOKBeforePriority(t *testing.T) { h := newHarness(t, Config{MaxEventsPerSec: 1}) diff --git a/pkg/waylog/v2/step.go b/pkg/waylog/v2/step.go index 56ae6da..cf56f2e 100644 --- a/pkg/waylog/v2/step.go +++ b/pkg/waylog/v2/step.go @@ -3,6 +3,7 @@ package waylogv2 import ( "context" "errors" + "fmt" "time" ) @@ -13,7 +14,7 @@ import ( // If ctx has no active Waylog request, fn runs and Step is a thin pass-through. // If name is empty, fn runs without opening a step (the empty name would // collide with the "no anchor yet" sentinel). -func Step[T any](ctx context.Context, name string, fn func(ctx context.Context) (T, error)) (T, error) { +func Step[T any](ctx context.Context, name string, fn func(ctx context.Context) (T, error)) (v T, err error) { r := requestFromContext(ctx) if r == nil || name == "" { return fn(ctx) @@ -21,10 +22,16 @@ func Step[T any](ctx context.Context, name string, fn func(ctx context.Context) startedAt := time.Now() startMS := int64(startedAt.Sub(r.tsStart) / 1e6) r.pushStep(name, startedAt, startMS) - v, err := fn(ctx) - dur := int64(time.Since(startedAt) / 1e6) - - r.closeStep(name, startMS, dur, err) + defer func() { + dur := int64(time.Since(startedAt) / 1e6) + if rec := recover(); rec != nil { + r.rememberPanicStep(name) + r.closeStep(name, startMS, dur, fmt.Errorf("panic: %v", rec)) + panic(rec) + } + r.closeStep(name, startMS, dur, err) + }() + v, err = fn(ctx) return v, err } @@ -71,6 +78,7 @@ func Fail(ctx context.Context, err *Error) { r.anchorStep = "request" } r.anchorCode = err.Code + r.anchorFromStepPanic = false } } @@ -96,6 +104,16 @@ func Suppress(ctx context.Context) { r.finalStatus = "" r.anchorStep = "" r.anchorCode = "" + r.anchorFromStepPanic = false + r.panicStepHint = "" +} + +func (r *request) rememberPanicStep(name string) { + r.mu.Lock() + if r.panicStepHint == "" { + r.panicStepHint = name + } + r.mu.Unlock() } func (r *request) pushStep(name string, startedAt time.Time, startMS int64) { diff --git a/pkg/waylog/v2/waylog.go b/pkg/waylog/v2/waylog.go index 183391a..f7c042d 100644 --- a/pkg/waylog/v2/waylog.go +++ b/pkg/waylog/v2/waylog.go @@ -75,6 +75,7 @@ type StatsSnapshot struct { LateCompletionAfterEmit int64 EventsDropped int64 DeliveryFailures int64 + EventsRejected int64 } // ErrAlreadyInitialized is returned by Init when a prior SDK is still alive @@ -147,6 +148,19 @@ func Init(cfg Config) error { devEnabled: devEnabled, active: make(map[*request]struct{}), } + + stateMu.Lock() + var previous *sdk + if state != nil { + state.mu.Lock() + n := len(state.active) + state.mu.Unlock() + if n > 0 { + stateMu.Unlock() + return fmt.Errorf("%w: %d in flight", ErrAlreadyInitialized, n) + } + previous = state + } if cfg.IngestURL != "" { delivery, err := transporthttp.New(transporthttp.Config{ IngestURL: cfg.IngestURL, @@ -156,22 +170,16 @@ func Init(cfg Config) error { InFlightCap: cfg.MaxInFlightBytes, }) if err != nil { + stateMu.Unlock() return err } s.delivery = delivery } - - stateMu.Lock() - defer stateMu.Unlock() - if state != nil { - state.mu.Lock() - n := len(state.active) - state.mu.Unlock() - if n > 0 { - return fmt.Errorf("%w: %d in flight", ErrAlreadyInitialized, n) - } - } state = s + stateMu.Unlock() + if previous != nil && previous.delivery != nil { + previous.delivery.Shutdown(0) + } return nil } @@ -226,6 +234,7 @@ func Stats() StatsSnapshot { LateCompletionAfterEmit: s.lateAfterEmit.Load(), EventsDropped: s.eventsDropped.Load() + deliveryDropped(s), DeliveryFailures: deliveryFailures(s), + EventsRejected: deliveryRejected(s), } } @@ -270,8 +279,19 @@ func deliveryFailures(s *sdk) int64 { return s.delivery.Failures() } +func deliveryRejected(s *sdk) int64 { + if s == nil || s.delivery == nil { + return 0 + } + return s.delivery.Rejected() +} + func resetForTest() { stateMu.Lock() + previous := state state = nil stateMu.Unlock() + if previous != nil && previous.delivery != nil { + previous.delivery.Shutdown(0) + } }