Observability Package

.github/workflows/build-and-deploy.yml

@@ -300,6 +300,8 @@ jobs:
         POSTGRES_DB=${{ needs.deploy-db.outputs.postgres-db }}
         POSTGRES_USER=${{ needs.deploy-db.outputs.postgres-user }}
         PGSSLMODE=${{ needs.deploy-db.outputs.postgres-sslmode }}
+        OTEL_SERVICE_NAME=stitch-api
+        OTEL_RESOURCE_ATTRIBUTES=deployment.name=${{ needs.resolve-context.outputs.deployment-name }},deployment.lane=${{ needs.resolve-context.outputs.deployment-lane }},service.version=${{ github.sha }}
       deployment-label: api
       # Keep staging / dress-rehearsal always-on; development scales to zero.
       min-replicas: ${{ needs.resolve-context.outputs.deployment-lane != 'development' && '1' || '' }}
@@ -327,6 +329,8 @@ jobs:
         AUTH_JWKS_URI=${{ needs.lane-config-validate.outputs.auth-jwks-uri }}
         ENTITY_LINKAGE_API_BASE_URL=${{ format('{0}/api/v1', needs.deploy-api.outputs.container-app-url) }}
         ENTITY_LINKAGE_FRONTEND_ORIGIN_URL=${{ needs.lane-config-validate.outputs.frontend-origin-url }}
+        OTEL_SERVICE_NAME=stitch-entity-linkage
+        OTEL_RESOURCE_ATTRIBUTES=deployment.name=${{ needs.resolve-context.outputs.deployment-name }},deployment.lane=${{ needs.resolve-context.outputs.deployment-lane }},service.version=${{ github.sha }}
       deployment-label: entity-linkage
       # Keep staging / dress-rehearsal always-on; development scales to zero.
       min-replicas: ${{ needs.resolve-context.outputs.deployment-lane != 'development' && '1' || '' }}
@@ -362,6 +366,8 @@ jobs:
         STITCH_LLM_AZURE_OPENAI_BASE_URL=${{ needs.lane-config-validate.outputs.stitch-llm-azure-openai-base-url }}
         STITCH_LLM_AZURE_OPENAI_MODEL=${{ needs.lane-config-validate.outputs.stitch-llm-azure-openai-model }}
         STITCH_LLM_AZURE_OPENAI_TIMEOUT_SECONDS=${{ needs.lane-config-validate.outputs.stitch-llm-azure-openai-timeout-seconds }}
+        OTEL_SERVICE_NAME=stitch-llm
+        OTEL_RESOURCE_ATTRIBUTES=deployment.name=${{ needs.resolve-context.outputs.deployment-name }},deployment.lane=${{ needs.resolve-context.outputs.deployment-lane }},service.version=${{ github.sha }}
       deployment-label: stitch-llm
       # Keep staging / dress-rehearsal always-on; development scales to zero.
       min-replicas: ${{ needs.resolve-context.outputs.deployment-lane != 'development' && '1' || '' }}

Makefile

@@ -107,9 +107,16 @@ pkg-test-ogsi:
 pkg-test-exact-ogsi:
 	$(MAKE) uv-test-target-exact PKG=stitch-ogsi TEST_PATH=packages/stitch-ogsi
 
-pkg-build: pkg-build-auth pkg-build-client pkg-build-models pkg-build-ogsi
-pkg-test: pkg-test-auth pkg-test-client pkg-test-models pkg-test-ogsi
-pkg-test-exact: pkg-test-exact-auth pkg-test-exact-client pkg-test-exact-models pkg-test-exact-ogsi
+pkg-build-observability:
+	$(UV) build --package stitch-observability
+pkg-test-observability:
+	$(MAKE) uv-test-target PKG=stitch-observability TEST_PATH=packages/stitch-observability
+pkg-test-exact-observability:
+	$(MAKE) uv-test-target-exact PKG=stitch-observability TEST_PATH=packages/stitch-observability
+
+pkg-build: pkg-build-auth pkg-build-client pkg-build-models pkg-build-ogsi pkg-build-observability
+pkg-test: pkg-test-auth pkg-test-client pkg-test-models pkg-test-ogsi pkg-test-observability
+pkg-test-exact: pkg-test-exact-auth pkg-test-exact-client pkg-test-exact-models pkg-test-exact-ogsi pkg-test-exact-observability
 
 # ---------------------------------------------------------------------
 # Deployments
@@ -291,6 +298,7 @@ follow-stack-logs:
 	pkg-build-client pkg-test-client pkg-test-exact-client \
 	pkg-build-models pkg-test-models pkg-test-exact-models \
 	pkg-build-ogsi pkg-test-ogsi pkg-test-exact-ogsi \
+	pkg-build-observability pkg-test-observability pkg-test-exact-observability \
 	\
 	# API
 	api-build api-test api-test-exact api-dev stack-api-dev \

deployments/PERFORMANCE.md

@@ -22,9 +22,15 @@ Two structured log streams, distinguished by the `logger` field:
 
 | Logger | Emitted | Key fields |
 |---|---|---|
-| `stitch.api.observability.request` | once per HTTP request (always) | `route`, `method`, `status_code`, `duration_ms`, `db_query_count`, `db_time_ms`, `request_id` |
+| `stitch.observability.request` | once per HTTP request (always) | `route`, `method`, `status_code`, `duration_ms`, `db_query_count`, `db_time_ms`, `request_id` |
 | `stitch.api.observability.query` | once per query above the slow threshold | `statement` (parameterized SQL, **no bound values**), `duration_ms`, `rowcount`, `route`, `request_id` |
 
+> The request summary is emitted by the shared `stitch.observability`
+> middleware, so it logs under `stitch.observability.request` (the API's
+> `db_query_count` / `db_time_ms` are grafted on by its subclass). The query
+> stream stays API-specific under `stitch.api.observability.query`. Filter on the
+> `observability` substring to capture both.
+
 `db_query_count` on a request is the N+1 detector; the `query` stream tells you
 *which* statement is expensive.
 
@@ -120,7 +126,7 @@ curl -s -o /dev/null -w '%{http_code}\n' \
 for i in $(seq 200); do
   curl -s -o /dev/null \
     -H "Authorization: Bearer $TOKEN" \
-    -H 'X-Perf-Scenario: vol=8k' \
+    -H 'X-Stitch-Perf-Scenario: vol=8k' \
     "http://localhost:8000/api/v1/oil-gas-fields/?page=1&page_size=50"
 done
 ```
@@ -137,7 +143,7 @@ For concurrency/throughput numbers, use a load tool if you have one installed
 ```bash
 hey -n 500 -c 20 \
   -H "Authorization: Bearer $TOKEN" \
-  -H "X-Perf-Scenario: vol=8k" \
+  -H "X-Stitch-Perf-Scenario: vol=8k" \
   "http://localhost:8000/api/v1/oil-gas-fields/?page=1&page_size=50"
 ```
 
@@ -201,7 +207,7 @@ az monitor log-analytics query \
     | where ContainerName_s == "api"
     | where TimeGenerated > ago(1h)
     | extend p = parse_json(Log_s)
-    | where tostring(p.logger) startswith "stitch.api.observability"
+    | where tostring(p.logger) contains "observability"
     | project line = Log_s' \
   -o tsv > /tmp/prod-events.jsonl
 ```
@@ -270,7 +276,7 @@ ROUTES — top 3 by total
 ## Comparing variants (data volume / params)
 
 To see how the *same* query behaves under different conditions, **tag each batch
-of traffic** with an `X-Perf-Scenario: <label>` request header. The label is
+of traffic** with an `X-Stitch-Perf-Scenario: <label>` request header. The label is
 recorded on every request *and* query event it triggers, so a single log
 captures all variants and the analyzer compares them with `--group-by scenario`.
 No log slicing, no separate files.
@@ -299,7 +305,7 @@ rows, so you can build up a volume ladder on a live stack.
 
    ```bash
    for i in $(seq 200); do
-     curl -s -o /dev/null -H 'X-Perf-Scenario: vol=1k' \
+     curl -s -o /dev/null -H 'X-Stitch-Perf-Scenario: vol=1k' \
        "http://localhost:8000/api/v1/oil-gas-fields/?page=1&page_size=50"
    done
    ```
@@ -316,7 +322,7 @@ rows, so you can build up a volume ladder on a live stack.
 
    ```bash
    for i in $(seq 200); do
-     curl -s -o /dev/null -H 'X-Perf-Scenario: vol=50k' \
+     curl -s -o /dev/null -H 'X-Stitch-Perf-Scenario: vol=50k' \
        "http://localhost:8000/api/v1/oil-gas-fields/?page=1&page_size=50"
    done
    ```
@@ -354,7 +360,7 @@ own label — the param values are a natural label:
 ```bash
 for ps in 50 500; do
   for i in $(seq 200); do
-    curl -s -o /dev/null -H "X-Perf-Scenario: page_size=$ps" \
+    curl -s -o /dev/null -H "X-Stitch-Perf-Scenario: page_size=$ps" \
       "http://localhost:8000/api/v1/oil-gas-fields/?page=1&page_size=$ps"
   done
 done
@@ -366,7 +372,7 @@ The `--group-by scenario` view breaks each query/route down by label, so
 `page_size=50` and `page_size=500` sit side by side even though they hit the
 same route template.
 
-> The `X-Perf-Scenario` label is opaque to the server (truncated to 80 chars)
+> The `X-Stitch-Perf-Scenario` label is opaque to the server (truncated to 80 chars)
 > and recorded only when sent, so it's safe to leave the feature in place — it
 > costs nothing on untagged production traffic.
 

deployments/api/pyproject.toml

@@ -10,14 +10,16 @@ dependencies = [
   "fastapi[standard-no-fastapi-cloud-cli]>=0.139.2",
   "greenlet>=3.3.0",
   "psycopg[binary]>=3.3.4",
+  # opentelemetry-sdk stays: the API imports `opentelemetry.trace` directly
+  # (request_logging) and its tests use opentelemetry.sdk. The OTLP exporter and
+  # the FastAPI/SQLAlchemy instrumentors now come transitively via
+  # stitch-observability (which owns tracing setup + instrumentation).
   "opentelemetry-sdk>=1.30.0",
-  "opentelemetry-exporter-otlp-proto-grpc>=1.30.0",
-  "opentelemetry-instrumentation-fastapi>=0.65b0",
-  "opentelemetry-instrumentation-sqlalchemy>=0.65b0",
   "pydantic-settings>=2.12.0",
   "sqlalchemy>=2.0.44",
   "stitch-auth",
   "stitch-models",
+  "stitch-observability[asgi]",
   "stitch-ogsi",
 ]
 
@@ -47,4 +49,5 @@ addopts = ["-v", "--strict-markers", "--tb=short"]
 [tool.uv.sources]
 stitch-auth = { workspace = true }
 stitch-models = { workspace = true }
+stitch-observability = { workspace = true }
 stitch-ogsi = { workspace = true }

deployments/api/src/stitch/api/db/config.py

@@ -10,7 +10,9 @@
     create_async_engine,
 )
 
-from stitch.api.observability import instrument_sqlalchemy, register_query_timing
+from stitch.observability import setup_sqlalchemy_tracing
+
+from stitch.api.observability import register_query_timing
 from stitch.api.settings import get_settings
 
 
@@ -59,8 +61,10 @@ def get_engine() -> AsyncEngine:
         log_all_queries=settings.log_all_queries,
     )
     # Per-query spans (separate from the aggregate timing listener above).
-    if settings.otel_enabled and settings.otel_traces_exporter != "none":
-        instrument_sqlalchemy(engine.sync_engine)
+    # get_engine is @lru_cache'd, so this instruments the engine once per
+    # process — SQLAlchemyInstrumentor is not safely re-entrant per engine.
+    # setup_sqlalchemy_tracing is a no-op when tracing is disabled.
+    setup_sqlalchemy_tracing(engine.sync_engine, settings=settings)
     return engine
 
 

deployments/api/src/stitch/api/main.py

@@ -6,10 +6,17 @@
 from opentelemetry.sdk.trace import TracerProvider
 from sqlalchemy.exc import OperationalError
 from starlette.status import HTTP_503_SERVICE_UNAVAILABLE
+from stitch.observability import (
+    configure_logging,
+    configure_tracing,
+    instrument_fastapi,
+    instrument_httpx,
+    resource_attributes_from_env,
+    shutdown_tracing,
+)
 from .middleware import register_middlewares
 from .db.config import dispose_engine
 from .auth import validate_auth_config_at_startup
-from .observability import configure_logging, configure_tracing, instrument_fastapi
 from .settings import Settings, get_settings
 
 from .routers.auth import router as auth_router
@@ -32,51 +39,87 @@ async def lifespan(app: FastAPI):
     app.state.auth_config_validated = True
     yield
     await dispose_engine()
-    tracer_provider = getattr(app.state, "tracer_provider", None)
-    if tracer_provider is not None:
-        # Flush any buffered spans (BatchSpanProcessor) before exit.
-        tracer_provider.shutdown()
+    # Flush any buffered spans (BatchSpanProcessor) before exit, using the
+    # provider this app was built with (attached in create_app), so a
+    # factory-built app shuts down its own provider rather than a module global.
+    # No-op if tracing is disabled (provider is None).
+    shutdown_tracing(getattr(app.state, "tracer_provider", None))
+
+
+# Global exception handler
+# - this will catch all exceptions of this type, incl. things like db constraint
+#   violations
+# - we can refine and narrow the scope at a later point
+async def db_unavailable_handler(_request: Request, _exc: OperationalError):
+    return JSONResponse(
+        status_code=HTTP_503_SERVICE_UNAVAILABLE,
+        content={"detail": "Database unavailable."},
+    )
 
 
 def create_app(
     settings: Settings, *, tracer_provider: TracerProvider | None
 ) -> FastAPI:
-    """Assemble the FastAPI application: middlewares, instrumentation, routers.
+    """Assemble the FastAPI application: middlewares, instrumentation, routers,
+    exception handlers.
 
-    ``tracer_provider`` is the value from :func:`configure_tracing` (``None`` when
-    tracing is disabled); when set, the app is auto-instrumented. Note that
-    ``FastAPIInstrumentor.instrument_app`` wraps its ASGI middleware *around the
-    whole user middleware stack* (CORS included), so the order of the calls below
-    does not affect the OpenTelemetry-vs-CORS layering.
+    Single source of truth for app assembly — both the module-level singleton
+    and the instrumentation tests build through here, so the two can't drift.
 
-    Extracted as a factory so tests can build an instrumented app; the shared
-    module-level ``app`` runs uninstrumented under the test env.
+    ``tracer_provider`` mirrors the return of the shared tracing setup (``None``
+    when tracing is disabled); when set, the app is auto-instrumented via
+    :func:`instrument_fastapi`. It is also stored on ``app.state.tracer_provider``
+    so ``lifespan`` flushes the provider this app was built with rather than a
+    module global. Note that ``FastAPIInstrumentor.instrument_app`` wraps its ASGI
+    middleware *around the whole user middleware stack* (CORS included), so the
+    order of the calls below does not affect the OpenTelemetry-vs-CORS layering.
     """
     application = FastAPI(lifespan=lifespan)
-    # Stash the provider on the app so lifespan can shut down the one this
-    # instance was actually built with, rather than a module-level global.
     application.state.tracer_provider = tracer_provider
     register_middlewares(application=application, settings=settings)
     if tracer_provider is not None:
         instrument_fastapi(application)
     application.include_router(base_router)
+    application.add_exception_handler(OperationalError, db_unavailable_handler)
     return application
 
 
 settings = get_settings()
 
-configure_logging(level=settings.log_level, log_format=settings.log_format)
-_tracer_provider = configure_tracing(settings)
+configure_logging(
+    level=settings.log_level,
+    log_format=settings.log_format,
+    # Stamp the same deployment metadata (deployment.name / lane / service.version)
+    # onto every log record that the tracing SDK stamps on spans, so logs and
+    # traces are comparable across deployments/PRs. Sourced from the shared
+    # OTEL_RESOURCE_ATTRIBUTES / OTEL_SERVICE_NAME env.
+    resource_attributes=resource_attributes_from_env(),
+)
 
+# Unlike entity-linkage / stitch-llm (which call the shared ``setup_fastapi_tracing``
+# one-shot), the API splits tracing *configuration* from app *assembly*: configure
+# the global provider here, then let ``create_app`` own ``instrument_fastapi``. That
+# keeps a single app-assembly path — shared with the instrumentation tests, which
+# pass their own provider — and avoids double-instrumenting. SQLAlchemy per-query
+# spans are set up separately in db/config.py, since the engine is created lazily.
+_tracer_provider = configure_tracing(
+    service_name="stitch-api",
+    enabled=settings.otel_enabled,
+    exporter=settings.otel_traces_exporter,
+    otlp_endpoint=settings.otel_exporter_otlp_endpoint,
+    sample_ratio=settings.otel_sample_ratio,
+    # None (not "unknown") when unset, so an env-provided service.version via
+    # OTEL_RESOURCE_ATTRIBUTES isn't clobbered by a placeholder.
+    version=settings.app_version,
+    environment=settings.environment_name,
+)
 app = create_app(settings, tracer_provider=_tracer_provider)
 
-
-# Global exception handler
-# - this will catch all exceptions of this type, incl. things like db constraint violations
-# - we can refine and narrow the scope at a a later point
-@app.exception_handler(OperationalError)
-async def db_unavailable_handler(_request: Request, _exc: OperationalError):
-    return JSONResponse(
-        status_code=HTTP_503_SERVICE_UNAVAILABLE,
-        content={"detail": "Database unavailable."},
-    )
+# Instrument outbound httpx as a process-global step (kept out of ``create_app``
+# so the instrumentation tests, which build through it with a real provider,
+# don't patch httpx for their test client). The API makes few/no outbound httpx
+# calls today, so this is close to a no-op — but when it does make one (e.g. a
+# JWKS fetch, or a future downstream call), it's captured as a client span and
+# carries ``traceparent`` for free, rather than being a silent gap in the trace.
+if _tracer_provider is not None:
+    instrument_httpx()

deployments/api/src/stitch/api/observability/__init__.py

@@ -1,28 +1,24 @@
-"""Performance instrumentation and tracing for the Stitch API.
+"""Performance instrumentation for the Stitch API.
 
-Two complementary layers:
+The generic machinery — tracing setup, structured logging, and the
+``RequestContextMiddleware`` that establishes per-request context and tags the
+active span — lives in the shared ``stitch.observability`` package and is wired
+up directly in :mod:`stitch.api.main` / :mod:`stitch.api.db.config`. This
+subpackage holds the API-specific pieces layered on top:
 
-* Structured-log timing — a SQLAlchemy event listener at the single engine
-  chokepoint (:mod:`query_timing`) and a request-timing middleware
-  (:mod:`request_logging`), both emitting through :mod:`sinks` to stdout, where
+* Structured-log query timing — a SQLAlchemy event listener at the single engine
+  chokepoint (:mod:`query_timing`) emitting through :mod:`sinks` to stdout, where
   Azure Container Apps forwards it to Log Analytics. Always on, independent of
   trace sampling.
-* OpenTelemetry tracing (:mod:`tracing`) — FastAPI / SQLAlchemy
-  auto-instrumentation producing spans, exported over OTLP to a collector or
-  logged to stdout (``OTEL_TRACES_EXPORTER``). The request middleware copies the
-  request id / scenario onto the active span so the two layers correlate.
+* Per-request DB aggregates — :mod:`request_logging` extends the shared
+  ``RequestContextMiddleware`` to add this request's query count / time to the
+  request-summary log event.
 """
 
-from .logging_config import configure_logging
 from .query_timing import register_query_timing
 from .request_logging import RequestTimingMiddleware
-from .tracing import configure_tracing, instrument_fastapi, instrument_sqlalchemy
 
 __all__ = [
-    "configure_logging",
     "register_query_timing",
     "RequestTimingMiddleware",
-    "configure_tracing",
-    "instrument_fastapi",
-    "instrument_sqlalchemy",
 ]