diff --git a/.env.example b/.env.example index c8c2ea65..0782e2a4 100644 --- a/.env.example +++ b/.env.example @@ -144,6 +144,40 @@ LANGFUSE_ENABLED=true LANGFUSE_PUBLIC_KEY=your-langfuse-public-key LANGFUSE_SECRET_KEY=your-langfuse-secret-key LANGFUSE_HOST=https://cloud.langfuse.com +# Required for Grafana dashboards: keep these trace fields enabled so dashboard +# filters and breakdowns can read ticket/project/workflow labels from Langfuse. +# Comma-separated list of trace attributes to set as Langfuse tags. +# Available values: ticket_key, ticket_type, project_id, workflow_step, repo, pr_number, ci_status, event_source, event_type, llm_model +LANGFUSE_TRACE_TAGS=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,llm_model +# Required for Grafana dashboards: project_id, ticket_type, workflow_step, and +# ticket_key must be present in metadata because dashboard ClickHouse queries use +# metadata['project_id'], metadata['ticket_type'], and metadata['workflow_step']. +# Comma-separated list of trace attributes to set as Langfuse metadata. +# Available values: ticket_key, ticket_type, project_id, workflow_step, repo, pr_number, ci_status, event_source, event_type, retry_count, system_prompt_length, llm_model +LANGFUSE_TRACE_METADATA=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,retry_count,system_prompt_length,llm_model + +# Grafana dashboard stack. These are used by docker-compose.yml and +# devtools/grafana/compose.grafana.yml. Prometheus/Redis defaults match this +# repo's compose files. +# +# Local self-hosted Langfuse defaults: +# - Start Langfuse from its compose project. +# - Run Grafana with devtools/grafana/compose.langfuse-network.yml so Grafana +# joins the Langfuse compose network. +# - The ClickHouse service is then reachable as clickhouse:9000. +GRAFANA_PORT=3010 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=grafana +LANGFUSE_DOCKER_NETWORK=langfuse_default +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_PORT=9000 +CLICKHOUSE_DATABASE=default +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_PASSWORD=clickhouse +PROMETHEUS_HOST=host.containers.internal +PROMETHEUS_PORT=9092 +REDIS_HOST=host.containers.internal +REDIS_PORT=6380 # OpenTelemetry distributed tracing (separate from Langfuse LLM tracing above) # Enable/disable OTLP trace export @@ -164,8 +198,8 @@ OTEL_SDK_DISABLED=true # - Production: your-registry.com/forge:v1.0.0 # Note: Use localhost/ prefix for local images to avoid podman short-name resolution prompts CONTAINER_IMAGE=localhost/forge-dev:latest -# Container execution timeout in seconds (default: 2 hours) -CONTAINER_TIMEOUT=7200 +# Container execution timeout in seconds (default: 30 minutes) +CONTAINER_TIMEOUT=1800 # Container resource limits CONTAINER_MEMORY=4g CONTAINER_CPUS=2 diff --git a/README.md b/README.md index 99335817..35f7c4e3 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,8 @@ tests/ # Unit and integration tests - **API server**: `http://localhost:8000/metrics` - **Worker**: `http://localhost:8001/metrics` +- **Prometheus UI**: `http://localhost:9092` +- **Grafana dashboards**: `http://localhost:3010` when the compose `grafana` service is running Key metrics: - `forge_workflows_started_total` - Workflows started by type diff --git a/devtools/README.md b/devtools/README.md index bf034560..0f12d2e3 100644 --- a/devtools/README.md +++ b/devtools/README.md @@ -1,18 +1,29 @@ # Forge Developer Tools -Local development stack for running Forge services on the host with Prometheus scraping both the API and worker. +Local development stack for running Forge services on the host with Prometheus scraping both the API and worker, plus Grafana dashboards for Forge observability. ## Usage ```bash -# Start Redis + Prometheus (scrapes host-local processes) -docker compose -f devtools/docker-compose.dev.yml up -d +# Start Redis + Prometheus + Grafana (scrapes host-local processes) +docker compose --env-file .env -f devtools/docker-compose.dev.yml up -d # In separate terminals, start the local services: uv run uvicorn forge.main:app --reload --port 8000 --host 0.0.0.0 uv run forge worker ``` +**With self-hosted Langfuse** — add the network override so Grafana can reach ClickHouse: + +```bash +docker compose --env-file .env \ + -f devtools/docker-compose.dev.yml \ + -f devtools/grafana/compose.langfuse-network.yml \ + up -d +``` + +> `compose.langfuse-network.yml` requires the `langfuse_default` Docker network to already exist (i.e. Langfuse must be running). Without it the entire stack will fail to start. Omit this file if you are not using self-hosted Langfuse — the Prometheus and Redis datasources work without it. + ## Endpoints | Service | URL | @@ -22,11 +33,16 @@ uv run forge worker | Worker metrics | http://localhost:8001/metrics | | Redis | redis://localhost:6380/0 | | Prometheus | http://localhost:9092 | +| Grafana | http://localhost:3010 | ## How it works `prometheus.dev.yml` targets `host.docker.internal` which resolves to the host machine from inside the Prometheus container. The `extra_hosts: host.docker.internal:host-gateway` entry in `docker-compose.dev.yml` enables this on Linux/Fedora. +Grafana provisions dashboards from `devtools/grafana/dashboards/` and datasources from `devtools/grafana/provisioning/`. The dashboards expect Forge to emit Langfuse trace tags and metadata for `ticket_key`, `ticket_type`, `project_id`, and `workflow_step`; `.env.example` enables those fields. + +For local self-hosted Langfuse, the `compose.langfuse-network.yml` override joins Grafana to the Langfuse compose network so it can reach ClickHouse at `clickhouse:9000`. Set `LANGFUSE_DOCKER_NETWORK` if your Langfuse compose network is not `langfuse_default`. + To reload Prometheus config without restarting: ```bash curl -X POST http://localhost:9092/-/reload diff --git a/devtools/docker-compose.dev.yml b/devtools/docker-compose.dev.yml index 6122f6c3..5b452801 100644 --- a/devtools/docker-compose.dev.yml +++ b/devtools/docker-compose.dev.yml @@ -11,6 +11,7 @@ version: "3.9" # uv run forge worker # # Prometheus dashboard: http://localhost:9092 +# Grafana dashboards: http://localhost:3010 (admin / grafana) services: redis: @@ -41,6 +42,34 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + grafana: + image: grafana/grafana:latest + ports: + - "${GRAFANA_PORT:-3010}:3000" + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-grafana} + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource,redis-datasource + - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-clickhouse} + - CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-9000} + - CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-default} + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-clickhouse} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse} + - PROMETHEUS_HOST=prometheus + - PROMETHEUS_PORT=9090 + - REDIS_HOST=redis + - REDIS_PORT=6379 + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro,z + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro,z + depends_on: + - prometheus + - redis + extra_hosts: + - "host.containers.internal:host-gateway" + volumes: redis-data: prometheus-data: + grafana-data: diff --git a/devtools/grafana/README.md b/devtools/grafana/README.md new file mode 100644 index 00000000..891b5cd5 --- /dev/null +++ b/devtools/grafana/README.md @@ -0,0 +1,213 @@ +# Forge Grafana Dashboards + +Grafana provisioning for Forge observability dashboards. The stack is +preconfigured for Langfuse ClickHouse, Forge Prometheus, and Forge Redis. + +## Start + +When running the full Forge compose stack: + +```bash +docker compose --env-file .env \ + -f docker-compose.yml \ + -f devtools/grafana/compose.langfuse-network.yml \ + up -d grafana +``` + +When running Forge services locally on the host with the devtools stack: + +```bash +docker compose --env-file .env \ + -f devtools/docker-compose.dev.yml \ + -f devtools/grafana/compose.langfuse-network.yml \ + up -d +``` + +For dashboard-only development against already-running Prometheus/Redis/ClickHouse: + +```bash +docker compose --env-file .env \ + -f devtools/grafana/compose.grafana.yml \ + -f devtools/grafana/compose.langfuse-network.yml \ + up -d +``` + +UI (default port): - log in as **admin / grafana**. + +## Required Forge Trace Fields + +The dashboards use Forge's configurable Langfuse trace fields for filters and +workflow-step breakdowns. Configure these in `.env` before running Forge: + +```bash +LANGFUSE_TRACE_TAGS=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,llm_model +LANGFUSE_TRACE_METADATA=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,retry_count,system_prompt_length,llm_model +``` + +## Local Self-Hosted Langfuse + +The default ClickHouse settings in `.env.example` match the standard local +Langfuse compose stack: + +```bash +LANGFUSE_DOCKER_NETWORK=langfuse_default +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_PORT=9000 +CLICKHOUSE_DATABASE=default +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_PASSWORD=clickhouse +``` + +Use `compose.langfuse-network.yml` in the `docker compose` command so Grafana +joins that network. If your Langfuse compose project uses a different network +name, set `LANGFUSE_DOCKER_NETWORK` accordingly. + +## Tear down + +```bash +docker compose --env-file .env -f devtools/grafana/compose.grafana.yml down -v +``` + +The `-v` flag removes the `grafana-data` volume, wiping dashboards and state. + +--- + +## Grafana MCP Server (Claude Code) + +The [grafana/mcp-grafana](https://github.com/grafana/mcp-grafana) server lets +Claude Code query datasources, read and write dashboards, manage alerts, and +navigate your Grafana instance through natural language. + +### 1. Create a service account token + +After starting the stack, create a token via the API: + +```bash +# Create a service account +SA_ID=$(curl -sf -u admin:grafana -X POST http://localhost:3010/api/serviceaccounts \ + -H 'Content-Type: application/json' \ + -d '{"name":"claude-code","role":"Editor"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + +# Generate a token for it +curl -sf -u admin:grafana -X POST http://localhost:3010/api/serviceaccounts/$SA_ID/tokens \ + -H 'Content-Type: application/json' \ + -d '{"name":"claude-code-token"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['key'])" +``` + +Save the printed token. + +### 2. Install mcp-grafana + +`mcp-grafana` is distributed as a Go binary and via PyPI. The easiest path if +you already have `uv` installed (which this project requires): + +```bash +uv tool install mcp-grafana +``` + +### 3. Add the MCP server to Claude Code + +Replace `` with the token from step 1. + +**Local scope** (current project only): + +```bash +claude mcp add-json "grafana" \ + '{"command":"uvx","args":["mcp-grafana"],"env":{"GRAFANA_URL":"http://localhost:3010","GRAFANA_SERVICE_ACCOUNT_TOKEN":""}}' +``` + +**User scope** (available across all your projects): + +```bash +claude mcp add-json "grafana" --scope user \ + '{"command":"uvx","args":["mcp-grafana"],"env":{"GRAFANA_URL":"http://localhost:3010","GRAFANA_SERVICE_ACCOUNT_TOKEN":""}}' +``` + +Verify the server is registered: + +```bash +claude mcp list +``` + +### 4. Verify the connection + +Start a Claude Code session and ask: + +> List my Grafana dashboards. + +Claude should respond with the list of the project's dashboards. + +--- + +### Reconfigure with a new token + +If you recreate the stack (`down -v`) or rotate the token, update the MCP +server with the new value: + +**Local scope** (current project only): + +```bash +claude mcp remove grafana +claude mcp add-json "grafana" \ + '{"command":"uvx","args":["mcp-grafana"],"env":{"GRAFANA_URL":"http://localhost:3010","GRAFANA_SERVICE_ACCOUNT_TOKEN":""}}' +``` + +**User scope** (available across all your projects): + +```bash +claude mcp remove grafana +claude mcp add-json "grafana" --scope user \ + '{"command":"uvx","args":["mcp-grafana"],"env":{"GRAFANA_URL":"http://localhost:3010","GRAFANA_SERVICE_ACCOUNT_TOKEN":""}}' +``` + +### Notes + +- The token is tied to the `grafana-data` volume. Running `down -v` destroys + the service account - repeat steps 1, 3, and the reconfigure step above after + recreating the stack. +- The **Editor** role is sufficient for most dashboard to work. Use **Admin** if + you need to manage datasources or users through Claude. +- To restrict Claude to read-only operations, add `"--disable-write"` to the + `args` array in step 3. + +--- + +## Dashboards + +Dashboard JSON files live in `devtools/grafana/dashboards/` and are version controlled. On startup, Grafana provisions them automatically. + +Sub-folders under `dashboards/` become Grafana folders. + +### Dashboard Inventory + +| Dashboard | Primary use | Datasources | +|-----------|-------------|-------------| +| Forge Operations Dashboard | Runtime health, webhooks, queues, retry/dead-letter depth | Prometheus, Redis | +| Forge Ticket Execution Dashboard | Single-ticket trace timeline, step cost, tokens, latency, agent calls | ClickHouse | +| Forge Cost Dashboard | Cost by project, ticket type, model, day, and expensive tickets | ClickHouse | +| Forge Agent Performance Dashboard | Agent invocation rate, duration, latency, prompt size, slow calls | Prometheus, ClickHouse | +| Forge Workflow Funnel Dashboard | Workflow starts/completions/failures, step coverage, rework loops | Prometheus, ClickHouse | +| Forge CI and Review Dashboard | CI fix attempts, approvals, revisions, PR/CI trace signals | Prometheus, ClickHouse | +| Forge Model Usage Dashboard | Calls, cost, tokens, and workflow-step usage by model | ClickHouse | +| Forge Observability Health Dashboard | Prometheus target health and trace metadata coverage | Prometheus, ClickHouse | +| Forge Business Dashboard | Business-level cost and throughput summaries | ClickHouse | +| Forge Engineering Dashboard | Mixed engineering views across runtime, alerts, cost, and latency | ClickHouse, Prometheus, Redis | +| Forge Issue Detail | Detailed per-issue trace and cost drill-down | ClickHouse, Prometheus | + +### Workflow + +Edited dashboards in the Grafana UI need to be synced back to the repo. Grafana holds live edits in its internal database; however, the local JSON files are the source of truth for version control. On a fresh stack (`down -v && up`), Grafana re-provisions the dashboards from the files. + +**Iterate on a dashboard:** + +1. Edit the dashboard in the Grafana UI - manually or through the mcp server +2. When happy with the changes, ask the AI to sync it back - e.g. _"save the dashboard back to the source file"_. +3. Claude uses the MCP server to fetch the current dashboard JSON and overwrites the local file. +4. Commit the updated file. + +**Create a new dashboard:** + +1. Ask Claude to create it - e.g. _"create a dashboard called Trace Volume with a time series panel"_. +2. Claude creates it via the MCP server (appears in the UI immediately). +3. Ask Claude to save it to `devtools/grafana/dashboards/.json`. +4. Commit the file - it will be provisioned automatically on next stack start. diff --git a/devtools/grafana/compose.grafana.yml b/devtools/grafana/compose.grafana.yml new file mode 100644 index 00000000..4b26bc79 --- /dev/null +++ b/devtools/grafana/compose.grafana.yml @@ -0,0 +1,41 @@ +# Ephemeral Grafana for Forge dashboard development. +# +# Usage: +# docker compose --env-file .env -f devtools/grafana/compose.grafana.yml up -d +# +# UI: http://localhost:${GRAFANA_PORT:-3010} (admin / grafana) +# +# This standalone compose file expects Prometheus, Redis, and Langfuse ClickHouse +# to be reachable from the Grafana container. For local self-hosted Langfuse, +# add compose.langfuse-network.yml so the default ClickHouse host "clickhouse" +# resolves on the Langfuse compose network. +name: forge-grafana + +services: + grafana: + image: grafana/grafana:latest + ports: + - "${GRAFANA_PORT:-3010}:3000" + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-grafana} + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource,redis-datasource + - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-clickhouse} + - CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-9000} + - CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-default} + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-clickhouse} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse} + - PROMETHEUS_HOST=${PROMETHEUS_HOST:-host.containers.internal} + - PROMETHEUS_PORT=${PROMETHEUS_PORT:-9092} + - REDIS_HOST=${REDIS_HOST:-host.containers.internal} + - REDIS_PORT=${REDIS_PORT:-6380} + volumes: + - grafana-data:/var/lib/grafana + - ./provisioning:/etc/grafana/provisioning:ro,z + - ./dashboards:/var/lib/grafana/dashboards:ro,z + extra_hosts: + - "host.containers.internal:host-gateway" + +volumes: + grafana-data: diff --git a/devtools/grafana/compose.langfuse-network.yml b/devtools/grafana/compose.langfuse-network.yml new file mode 100644 index 00000000..6b5a8cc3 --- /dev/null +++ b/devtools/grafana/compose.langfuse-network.yml @@ -0,0 +1,24 @@ +# Optional compose override for local self-hosted Langfuse. +# +# Use this when Langfuse is running from its own compose project and exposes +# ClickHouse on the Langfuse compose network. The default network name matches a +# Langfuse checkout started as project "langfuse"; override with +# LANGFUSE_DOCKER_NETWORK if your compose project name is different. +# +# Example: +# docker compose --env-file .env \ +# -f docker-compose.yml \ +# -f devtools/grafana/compose.langfuse-network.yml \ +# up -d grafana + +services: + grafana: + networks: + - default + - langfuse + +networks: + default: {} + langfuse: + name: ${LANGFUSE_DOCKER_NETWORK:-langfuse_default} + external: true diff --git a/devtools/grafana/dashboards/forge-agent-performance.json b/devtools/grafana/dashboards/forge-agent-performance.json new file mode 100644 index 00000000..e569c5db --- /dev/null +++ b/devtools/grafana/dashboards/forge-agent-performance.json @@ -0,0 +1,658 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-agent-performance", + "namespace": "default", + "uid": "forge-agent-performance" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Agent Invocations", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_agent_invocations_total[$__rate_interval])) by (task_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Agent Duration", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "rate(forge_agent_duration_seconds_sum[$__rate_interval]) / rate(forge_agent_duration_seconds_count[$__rate_interval])" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Latency by Model", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT provided_model_name as model, round(avg(dateDiff('second', start_time, coalesce(end_time, start_time))), 2) as avg_latency_s FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY model ORDER BY avg_latency_s DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Tokens by Workflow Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['workflow_step'] IN ($workflow_step) GROUP BY step ORDER BY input_tokens DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Prompt Size by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT metadata['workflow_step'] as step, round(avg(toInt64OrZero(metadata['system_prompt_length'])), 0) as avg_prompt_chars FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['workflow_step'] IN ($workflow_step) GROUP BY step ORDER BY avg_prompt_chars DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Slowest Agent Calls", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.session_id as issue, t.metadata['workflow_step'] as step, o.provided_model_name as model, dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time)) as latency_s, round(o.total_cost, 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) ORDER BY latency_s DESC LIMIT 50" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Runtime Metrics", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Trace Performance", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Slow Calls", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "agents", + "performance", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Agent Performance Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-business.json b/devtools/grafana/dashboards/forge-business.json new file mode 100644 index 00000000..cefc87bc --- /dev/null +++ b/devtools/grafana/dashboards/forge-business.json @@ -0,0 +1,1291 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-business", + "namespace": "default", + "uid": "ccbcaaa1-7e80-4dbc-b8bb-99364c3b859e", + "resourceVersion": "1779151422257995", + "generation": 2, + "creationTimestamp": "2026-05-16T11:40:55Z", + "labels": { + "grafana.app/deprecatedInternalID": "1973036806004736" + }, + "annotations": { + "grafana.app/createdBy": "access-policy:service", + "grafana.app/managedBy": "classic-file-provisioning", + "grafana.app/managerAllowsEdits": "true", + "grafana.app/managerId": "default", + "grafana.app/sourceChecksum": "84db0f6d67a23188f459897aef46443e", + "grafana.app/sourcePath": "/var/lib/grafana/dashboards/forge-business.json", + "grafana.app/sourceTimestamp": "1779150775000", + "grafana.app/updatedBy": "access-policy:service", + "grafana.app/updatedTimestamp": "2026-05-18T17:23:17Z" + } + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Total Cost Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT $__timeInterval(t.timestamp) as time, sum(o.total_cost) as total_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY time ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "id": 24, + "title": "Cost by Project Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "timeseries", + "rawSql": "SELECT $__timeInterval(t.timestamp) as time, t.metadata['project_id'] as project, sum(o.total_cost) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY time, project ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 15, + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Cost by Model Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "timeseries", + "rawSql": "SELECT $__timeInterval(o.start_time) as time, o.provided_model_name as model, sum(o.total_cost) as cost FROM default.observations o FINAL WHERE $__timeFilter(o.start_time) AND o.provided_model_name IN ($model) AND o.trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id IN ($jira_issue) AND metadata['ticket_type'] IN ($ticket_type)) GROUP BY time, model ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 15, + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "id": 13, + "title": "Cost by Ticket Type Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "timeseries", + "rawSql": "SELECT $__timeInterval(t.timestamp) as time, t.metadata['ticket_type'] as ticket_type, sum(o.total_cost) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY time, ticket_type ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Avg Completion Time (Features)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "timeseries", + "rawSql": "SELECT toStartOfWeek(start_time) as time, round(avg(hours_elapsed), 1) as avg_hours FROM (SELECT t.session_id, min(t.timestamp) as start_time, dateDiff('hour', min(t.timestamp), max(t.timestamp)) as hours_elapsed FROM default.traces t FINAL WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] = 'Feature' GROUP BY t.session_id HAVING count(*) > 1) GROUP BY time ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "h", + "custom": { + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "id": 25, + "title": "Avg Completion Time (Bugs)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "timeseries", + "rawSql": "SELECT toStartOfWeek(start_time) as time, round(avg(hours_elapsed), 1) as avg_hours FROM (SELECT t.session_id, min(t.timestamp) as start_time, dateDiff('hour', min(t.timestamp), max(t.timestamp)) as hours_elapsed FROM default.traces t FINAL WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] = 'Bug' GROUP BY t.session_id HAVING count(*) > 1) GROUP BY time ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "h", + "custom": { + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "id": 18, + "title": "Issues (sorted by cost)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.session_id AS issue, t.metadata['project_id'] as project, any(t.metadata['ticket_type']) as type, count(DISTINCT t.id) as steps, round(sum(o.total_cost), 2) AS total_cost FROM default.traces t FINAL LEFT JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id, project ORDER BY total_cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total_cost" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "issue" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "Issue Detail", + "url": "/d/forge-issue-detail?var-jira_issue=${__data.fields.issue}" + } + ] + } + ] + } + ] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Total LLM Cost", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(sum(o.total_cost), 2) as value FROM default.observations o FINAL WHERE $__timeFilter(o.start_time) AND o.trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id IN ($jira_issue) AND metadata['ticket_type'] IN ($ticket_type))" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Avg Cost / Feature", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(avg(issue_cost), 2) as value FROM (SELECT t.session_id, sum(o.total_cost) as issue_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['ticket_type'] = 'Feature' AND t.session_id IN ($jira_issue) GROUP BY t.session_id)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Avg Cost / Bug Fix", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(avg(issue_cost), 2) as value FROM (SELECT t.session_id, sum(o.total_cost) as issue_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['ticket_type'] = 'Bug' AND t.session_id IN ($jira_issue) GROUP BY t.session_id)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Avg Time / Issue", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(avg(hours_elapsed), 1) as value FROM (SELECT t.session_id, dateDiff('hour', min(t.timestamp), max(t.timestamp)) as hours_elapsed FROM default.traces t FINAL WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "h" + }, + "overrides": [] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Ticket Count", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT uniqExact(session_id) as value FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND session_id IN ($jira_issue) AND metadata['ticket_type'] IN ($ticket_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Total Cost by Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['project_id'] as project, round(sum(o.total_cost), 2) as total_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY project ORDER BY total_cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "project" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Cost per Ticket (Top 10)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.session_id as issue, t.metadata['ticket_type'] as ticket_type, sum(o.total_cost) as total_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id, ticket_type ORDER BY total_cost DESC LIMIT 10" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "xField": "issue", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "\ud83d\udcb0 Executive KPIs", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 5, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 10, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 15, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "\ud83d\udcca Feature vs Bug Economics", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "\ud83d\udcc8 Cost Trends", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-24" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-13" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "\ud83d\udd04 Efficiency & Waste", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-25" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "\ud83d\udccb Issues Table", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 12, + "element": { + "kind": "ElementReference", + "name": "panel-18" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [ + { + "title": "\u2190 Engineering Dashboard", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/forge-engineering", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": false, + "keepTime": true + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "business" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "30m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Forge Business Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "CustomVariable", + "spec": { + "name": "ticket_type", + "query": "Feature,Bug", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Feature", + "value": "Feature" + }, + { + "selected": false, + "text": "Bug", + "value": "Bug" + } + ], + "multi": true, + "includeAll": true, + "label": "Ticket Type", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "model", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Model", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT provided_model_name FROM default.observations FINAL WHERE provided_model_name IS NOT NULL AND provided_model_name != '' ORDER BY provided_model_name" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT provided_model_name FROM default.observations FINAL WHERE provided_model_name IS NOT NULL AND provided_model_name != '' ORDER BY provided_model_name", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + }, + "status": { + "conversion": { + "failed": false, + "storedVersion": "v0alpha1" + } + } +} diff --git a/devtools/grafana/dashboards/forge-ci-review.json b/devtools/grafana/dashboards/forge-ci-review.json new file mode 100644 index 00000000..5405760e --- /dev/null +++ b/devtools/grafana/dashboards/forge-ci-review.json @@ -0,0 +1,653 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-ci-review", + "namespace": "default", + "uid": "forge-ci-review" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "CI Fix Attempts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_ci_fix_attempts_total) by (repo, result)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Approvals", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_approvals_total) by (stage)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Revision Requests", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_revisions_requested_total) by (stage)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "CI Status Trace Count", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT metadata['ci_status'] as ci_status, count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['ci_status'] != '' GROUP BY ci_status ORDER BY traces DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "PR Activity", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT session_id as issue, metadata['repo'] as repo, metadata['pr_number'] as pr_number, metadata['ci_status'] as ci_status, max(timestamp) as latest_trace FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND (metadata['repo'] != '' OR metadata['pr_number'] != '' OR metadata['ci_status'] != '') GROUP BY issue, repo, pr_number, ci_status ORDER BY latest_trace DESC LIMIT 100" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Review/CI Steps", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT metadata['workflow_step'] as step, count() as traces, uniqExact(session_id) as issues FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND (positionCaseInsensitive(metadata['workflow_step'], 'review') > 0 OR positionCaseInsensitive(metadata['workflow_step'], 'ci') > 0) GROUP BY step ORDER BY traces DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "CI and Gate Metrics", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Trace Signals", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "PR Activity", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "ci", + "review", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge CI and Review Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-cost.json b/devtools/grafana/dashboards/forge-cost.json new file mode 100644 index 00000000..eee6a311 --- /dev/null +++ b/devtools/grafana/dashboards/forge-cost.json @@ -0,0 +1,667 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-cost", + "namespace": "default", + "uid": "forge-cost" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Total Cost", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(sum(total_cost), 2) as value FROM default.observations FINAL WHERE $__timeFilter(start_time)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Cost by Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['project_id'] as project, round(sum(o.total_cost), 2) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['project_id'] IN ($project_id) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY project ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Cost by Ticket Type", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['ticket_type'] as ticket_type, round(sum(o.total_cost), 2) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['project_id'] IN ($project_id) GROUP BY ticket_type ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "piechart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Cost by Model", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT provided_model_name as model, round(sum(total_cost), 2) as cost FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY model ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Daily Cost", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT toStartOfDay(start_time) as time, round(sum(total_cost), 2) as cost FROM default.observations FINAL WHERE $__timeFilter(start_time) GROUP BY time ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Top Expensive Tickets", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.session_id as issue, t.metadata['project_id'] as project, t.metadata['ticket_type'] as ticket_type, round(sum(o.total_cost), 2) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY issue, project, ticket_type ORDER BY cost DESC LIMIT 20" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Cost Summary", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 18, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Cost Breakdown", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Outliers", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "cost", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Cost Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-engineering.json b/devtools/grafana/dashboards/forge-engineering.json new file mode 100644 index 00000000..dedbbe74 --- /dev/null +++ b/devtools/grafana/dashboards/forge-engineering.json @@ -0,0 +1,2184 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-engineering", + "namespace": "default", + "uid": "d3fc94fa-99b6-4b1e-bebe-b4fb9b8575e7", + "resourceVersion": "1779153130725011", + "generation": 55, + "creationTimestamp": "2026-05-16T11:40:55Z", + "labels": { + "grafana.app/deprecatedInternalID": "1973036843753472" + }, + "annotations": { + "grafana.app/createdBy": "access-policy:service", + "grafana.app/managedBy": "classic-file-provisioning", + "grafana.app/managerAllowsEdits": "true", + "grafana.app/managerId": "default", + "grafana.app/sourceChecksum": "59989cbc9e98b1ce5ab60ded451018df", + "grafana.app/sourcePath": "/var/lib/grafana/dashboards/forge-engineering.json", + "grafana.app/sourceTimestamp": "1779151016000", + "grafana.app/updatedBy": "user:afm8ei87lasjkf", + "grafana.app/updatedTimestamp": "2026-05-19T01:12:10Z" + } + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Iteration Count Distribution", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count(*) as trace_count FROM default.traces t FINAL WHERE $__timeFilter(t.timestamp) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "histogram", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 19 + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Token Ratio Distribution", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(sum(o.usage_details['input']) / greatest(sum(o.usage_details['output']), 1), 1) as token_ratio FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "histogram", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Total Alerts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "HGET forge:stats:alerts:summary total", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 78 + } + } + }, + "panel-22": { + "kind": "Panel", + "spec": { + "id": 22, + "title": "Cost Outliers", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.timestamp as time, t.session_id as issue, sum(CASE o.provided_model_name WHEN 'claude-opus-4-7' THEN o.usage_details['input'] * 0.000015 + o.usage_details['output'] * 0.000075 WHEN 'claude-opus-4-5@20251101' THEN o.usage_details['input'] * 0.000015 + o.usage_details['output'] * 0.000075 WHEN 'claude-sonnet-4-6' THEN o.usage_details['input'] * 0.000003 + o.usage_details['output'] * 0.000015 WHEN 'gemini-2.5-pro' THEN o.usage_details['input'] * 0.00000125 + o.usage_details['output'] * 0.00001 WHEN 'claude-haiku-4-5@20251001' THEN o.usage_details['input'] * 0.000001 + o.usage_details['output'] * 0.000005 ELSE 0 END) as cost, t.metadata['ticket_type'] as ticket_type FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.timestamp, t.session_id, ticket_type ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "custom": { + "drawStyle": "points", + "pointSize": 8 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "issue" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "Issue Detail", + "url": "/d/forge-issue-detail?var-jira_issue=${__data.fields.issue}" + } + ] + } + ] + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "id": 23, + "title": "Latency Outliers", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.timestamp as time, t.session_id as issue, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as latency_s, t.metadata['workflow_step'] as workflow_step FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) GROUP BY t.timestamp, t.session_id, t.id, workflow_step ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "points", + "pointSize": 8 + } + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "id": 24, + "title": "Avg Cost per Workflow Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(avg(o_agg.trace_cost), 4) as avg_cost FROM default.traces t FINAL JOIN (SELECT trace_id, sum(CASE provided_model_name WHEN 'claude-opus-4-7' THEN usage_details['input'] * 0.000015 + usage_details['output'] * 0.000075 WHEN 'claude-opus-4-5@20251101' THEN usage_details['input'] * 0.000015 + usage_details['output'] * 0.000075 WHEN 'claude-sonnet-4-6' THEN usage_details['input'] * 0.000003 + usage_details['output'] * 0.000015 WHEN 'gemini-2.5-pro' THEN usage_details['input'] * 0.00000125 + usage_details['output'] * 0.00001 WHEN 'claude-haiku-4-5@20251001' THEN usage_details['input'] * 0.000001 + usage_details['output'] * 0.000005 ELSE 0 END) as trace_cost FROM default.observations FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY avg_cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "id": 25, + "title": "Avg Latency by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(avg(o_agg.latency_s), 1) as avg_latency FROM default.traces t FINAL JOIN (SELECT trace_id, dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s FROM default.observations FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY avg_latency DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "id": 26, + "title": "Avg Iteration Cycles per Workflow Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(avg(step_traces), 1) as avg_cycles FROM (SELECT session_id, metadata['workflow_step'] as ws, count(*) as step_traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND session_id IN ($jira_issue) AND metadata['ticket_type'] IN ($ticket_type) AND metadata['workflow_step'] != '' GROUP BY session_id, ws) sub JOIN default.traces t FINAL ON sub.session_id = t.session_id AND sub.ws = t.metadata['workflow_step'] WHERE t.metadata['workflow_step'] != '' GROUP BY step ORDER BY avg_cycles DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 36 + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "id": 29, + "title": "Avg Token Efficiency (Input:Output Ratio) by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(sum(o.usage_details['input']) / greatest(sum(o.usage_details['output']), 1), 1) as ratio FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY ratio DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Critical", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "HGET forge:stats:alerts:summary critical", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + } + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 78 + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "id": 32, + "title": "Avg Phase Duration by Phase & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "rate(forge_phase_duration_seconds_sum{phase=~\"$workflow_step\"}[$__rate_interval]) / rate(forge_phase_duration_seconds_count{phase=~\"$workflow_step\"}[$__rate_interval])", + "legendFormat": "{{project_id}} / {{phase}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "id": 33, + "title": "Avg Agent Duration by Task & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "rate(forge_agent_duration_seconds_sum{task_type=~\"$workflow_step\"}[$__rate_interval]) / rate(forge_agent_duration_seconds_count{task_type=~\"$workflow_step\"}[$__rate_interval])", + "legendFormat": "{{project_id}} / {{task_type}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "id": 34, + "title": "Avg External API Latency by Service & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "rate(forge_external_api_latency_seconds_sum[$__rate_interval]) / rate(forge_external_api_latency_seconds_count[$__rate_interval])", + "legendFormat": "{{project_id}} / {{service}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "id": 35, + "title": "Workflow Throughput by Project & Type", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_started_total{ticket_type=~\"$ticket_type\"}) by (ticket_type)", + "legendFormat": "{{project_id}} / {{ticket_type}} started" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_completed_total{ticket_type=~\"$ticket_type\"}) by (ticket_type)", + "legendFormat": "{{project_id}} / {{ticket_type}} completed" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_failed_total{ticket_type=~\"$ticket_type\"}) by (ticket_type)", + "legendFormat": "{{project_id}} / {{ticket_type}} failed" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + } + } + }, + "panel-36": { + "kind": "Panel", + "spec": { + "id": 36, + "title": "Approvals vs Revisions by Stage & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_approvals_total{stage=~\"$workflow_step\"}) by (stage)", + "legendFormat": "{{project_id}} / {{stage}} approved" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_revisions_requested_total{stage=~\"$workflow_step\"}) by (stage)", + "legendFormat": "{{project_id}} / {{stage}} revision" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + } + } + }, + "panel-37": { + "kind": "Panel", + "spec": { + "id": 37, + "title": "CI Fix Attempts by Result & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_ci_fix_attempts_total) by (repo, result)", + "legendFormat": "{{project_id}} / {{result}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + } + } + }, + "panel-38": { + "kind": "Panel", + "spec": { + "id": 38, + "title": "Agent Invocations by Task & Project", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_agent_invocations_total{task_type=~\"$workflow_step\"}) by (task_type)", + "legendFormat": "{{project_id}} / {{task_type}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 69 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Warning", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "HGET forge:stats:alerts:summary warning", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "yellow" + } + ] + } + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 78 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Alerts by Type", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "command": "xrange", + "end": "+", + "keyName": "forge:alerts:stream", + "start": "-", + "type": "stream" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "groupBy", + "spec": { + "id": "groupBy", + "options": { + "fields": { + "alert_type": { + "aggregations": [], + "operation": "groupby" + }, + "severity": { + "aggregations": [ + "count" + ], + "operation": "aggregate" + } + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "alert_type" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Alerts Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "command": "ts.range", + "keyName": "forge:stats:alerts:ts:cost_outlier", + "legend": "Cost Outliers", + "type": "timeSeries" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "command": "ts.range", + "keyName": "forge:stats:alerts:ts:latency_outlier", + "legend": "Latency Outliers", + "type": "timeSeries" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Alerts List", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "command": "xrange", + "end": "+", + "keyName": "forge:alerts:stream", + "start": "-", + "type": "stream" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "organize", + "spec": { + "id": "organize", + "options": { + "excludeByName": { + "$streamId": true, + "$time": true, + "sigma": true, + "trace_ids": true + }, + "indexByName": { + "actual_value": 7, + "alert_id": 0, + "alert_type": 4, + "description": 6, + "fired_at": 9, + "issue_id": 1, + "project_id": 2, + "severity": 5, + "threshold": 8, + "ticket_type": 3 + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "issue_id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "Open Issue Detail", + "url": "/d/forge-issue-detail?var-jira_issue=${__data.fields.issue_id}" + } + ] + } + ] + } + ] + } + } + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 90 + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Cost Distribution", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT sum(CASE provided_model_name WHEN 'claude-opus-4-7' THEN usage_details['input'] * 0.000015 + usage_details['output'] * 0.000075 WHEN 'claude-opus-4-5@20251101' THEN usage_details['input'] * 0.000015 + usage_details['output'] * 0.000075 WHEN 'claude-sonnet-4-6' THEN usage_details['input'] * 0.000003 + usage_details['output'] * 0.000015 WHEN 'gemini-2.5-pro' THEN usage_details['input'] * 0.00000125 + usage_details['output'] * 0.00001 WHEN 'claude-haiku-4-5@20251001' THEN usage_details['input'] * 0.000001 + usage_details['output'] * 0.000005 ELSE 0 END) as issue_cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['ticket_type'] IN ($ticket_type) GROUP BY t.session_id" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "histogram", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Latency Distribution", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as trace_latency_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) GROUP BY t.session_id, t.id" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "histogram", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "⚡ Performance", + "collapse": true, + "layout": { + "kind": "TabsLayout", + "spec": { + "tabs": [ + { + "kind": "TabsLayoutTab", + "spec": { + "title": "Cost/Tokens", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-22" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-24" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 8, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-29" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 8, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + } + ] + } + } + } + }, + { + "kind": "TabsLayoutTab", + "spec": { + "title": "Code Quality", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [] + } + } + } + }, + { + "kind": "TabsLayoutTab", + "spec": { + "title": "Human Intervention", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + } + ] + } + } + } + }, + { + "kind": "TabsLayoutTab", + "spec": { + "title": "Time and Latency", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-23" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-25" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 8, + "width": 16, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-26" + } + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "🏥 System Health", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-35" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-32" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-33" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-34" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-36" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-37" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 24, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-38" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "🚨 Alerts", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 4, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 4, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 12, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [ + { + "title": "Business Dashboard →", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/forge-business", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": false, + "keepTime": true + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "engineering" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-7d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Forge Engineering Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "langfuse_host", + "query": "http://localhost:3000", + "current": { + "text": "http://localhost:3000", + "value": "http://localhost:3000" + }, + "label": "Langfuse Host", + "hide": "hideVariable", + "skipUrlSync": false + } + } + ] + }, + "status": { + "conversion": { + "failed": false, + "storedVersion": "v0alpha1" + } + } +} diff --git a/devtools/grafana/dashboards/forge-issue-detail.json b/devtools/grafana/dashboards/forge-issue-detail.json new file mode 100644 index 00000000..dd02972f --- /dev/null +++ b/devtools/grafana/dashboards/forge-issue-detail.json @@ -0,0 +1,1002 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-issue-detail", + "namespace": "default", + "uid": "599c08a0-61b1-482f-98bd-db7d73d6fb89", + "resourceVersion": "1779151422247012", + "generation": 2, + "creationTimestamp": "2026-05-16T11:40:55Z", + "labels": { + "grafana.app/deprecatedInternalID": "1973036877307904" + }, + "annotations": { + "grafana.app/createdBy": "access-policy:service", + "grafana.app/managedBy": "classic-file-provisioning", + "grafana.app/managerAllowsEdits": "true", + "grafana.app/managerId": "default", + "grafana.app/sourceChecksum": "1d3c69ac75d3bdb4ca6385e9077f2e8e", + "grafana.app/sourcePath": "/var/lib/grafana/dashboards/forge-issue-detail.json", + "grafana.app/sourceTimestamp": "1779150775000", + "grafana.app/updatedBy": "access-policy:service", + "grafana.app/updatedTimestamp": "2026-05-18T17:23:17Z" + } + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Cost by Workflow Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Token Usage by Step (Input vs Output)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY input_tokens DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "step" + }, + "fieldConfig": { + "defaults": { + "unit": "locale" + }, + "overrides": [] + } + } + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "id": 14, + "title": "Forge Metrics During Issue Lifecycle", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_phase_duration_seconds_sum[1h])) by (phase)", + "legendFormat": "phase_duration {{phase}}" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT $__timeInterval(o.start_time) as time, avg(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))) as avg_trace_latency_s FROM default.observations o FINAL WHERE o.trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}') GROUP BY time ORDER BY time" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Traces Table", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.id as id, o_agg.model, round(o_agg.latency_s, 1) as latency_s, round(o_agg.cost, 4) as cost, o_agg.total_tokens as tokens FROM default.traces t FINAL JOIN (SELECT trace_id, any(provided_model_name) as model, dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s, sum(total_cost) as cost, sum(usage_details['total']) as total_tokens FROM default.observations FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' ORDER BY t.timestamp" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cost" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "latency_s" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open in Langfuse", + "url": "${langfuse_host}/trace/${__data.fields.id}" + } + ] + } + ] + } + ] + } + } + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "id": 26, + "title": "Ticket", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "text", + "version": "", + "spec": { + "options": { + "mode": "html", + "content": "
${jira_issue}
" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Type", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT any(if(metadata['ticket_type'] = 'Feature', 1, 0)) as value FROM default.traces FINAL WHERE session_id = '${jira_issue}'" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Bug" + }, + "1": { + "text": "Feature" + } + } + } + ] + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Total Traces", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT uniqExact(id) as value FROM default.traces FINAL WHERE session_id = '${jira_issue}'" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Total Cost", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(sum(total_cost), 4) as value FROM default.observations FINAL WHERE trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}')" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Avg Latency", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT round(avg(latency_s), 1) as value FROM (SELECT dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s FROM default.observations FINAL WHERE trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}') GROUP BY trace_id)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Workflow Step Timeline (duration per step)", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY t.session_id, t.id, step ORDER BY min(o.start_time)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "xField": "step" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Issue KPIs", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-26" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 5, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 9, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 13, + "y": 0, + "width": 5, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 0, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Workflow Waterfall", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Cost & Token Breakdown", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 12, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 12, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prometheus Correlation", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-14" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Traces", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 12, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [ + { + "title": "\u2190 Engineering", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/forge-engineering", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": false, + "keepTime": true + }, + { + "title": "\u2190 Business", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/forge-business", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": false, + "keepTime": true + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "detail" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Forge Issue Detail", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "", + "value": "" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": false, + "includeAll": false, + "allowCustomValue": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "langfuse_host", + "query": "http://localhost:3000", + "current": { + "text": "http://localhost:3000", + "value": "http://localhost:3000" + }, + "label": "Langfuse Host", + "hide": "hideVariable", + "skipUrlSync": false + } + } + ] + }, + "status": { + "conversion": { + "failed": false, + "storedVersion": "v0alpha1" + } + } +} diff --git a/devtools/grafana/dashboards/forge-model-usage.json b/devtools/grafana/dashboards/forge-model-usage.json new file mode 100644 index 00000000..0e76d23e --- /dev/null +++ b/devtools/grafana/dashboards/forge-model-usage.json @@ -0,0 +1,589 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-model-usage", + "namespace": "default", + "uid": "forge-model-usage" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Calls by Model", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT provided_model_name as model, count() as calls FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY model ORDER BY calls DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Cost by Model", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT provided_model_name as model, round(sum(total_cost), 2) as cost FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY model ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Tokens by Model", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT provided_model_name as model, sum(usage_details['input']) as input_tokens, sum(usage_details['output']) as output_tokens FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY model ORDER BY input_tokens DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Model Usage Over Time", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT toStartOfDay(start_time) as time, provided_model_name as model, count() as calls FROM default.observations FINAL WHERE $__timeFilter(start_time) AND provided_model_name != '' GROUP BY time, model ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Model by Workflow Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, o.provided_model_name as model, count() as calls, round(sum(o.total_cost), 2) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['workflow_step'] IN ($workflow_step) AND o.provided_model_name != '' GROUP BY step, model ORDER BY calls DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Model Volume and Cost", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Trends", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Workflow Mapping", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "models", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Model Usage Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-observability-health.json b/devtools/grafana/dashboards/forge-observability-health.json new file mode 100644 index 00000000..42a5340d --- /dev/null +++ b/devtools/grafana/dashboards/forge-observability-health.json @@ -0,0 +1,777 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-observability-health", + "namespace": "default", + "uid": "forge-observability-health" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Prometheus Targets", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "up" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Last Trace Timestamp", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT metadata['project_id'] as project, max(timestamp) as latest_trace FROM default.traces FINAL WHERE metadata['project_id'] != '' GROUP BY project ORDER BY latest_trace DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Missing project_id", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['project_id'] = ''" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Missing ticket_type", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['ticket_type'] = ''" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Missing workflow_step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['workflow_step'] = ''" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Empty session_id", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND (session_id IS NULL OR session_id = '')" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Observation Orphans", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as observations_without_trace FROM default.observations o FINAL LEFT JOIN default.traces t FINAL ON o.trace_id = t.id WHERE $__timeFilter(o.start_time) AND t.id IS NULL" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Trace Field Coverage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT toStartOfDay(timestamp) as time, countIf(metadata['project_id'] != '') as project_id, countIf(metadata['ticket_type'] != '') as ticket_type, countIf(metadata['workflow_step'] != '') as workflow_step, countIf(session_id != '') as session_id FROM default.traces FINAL WHERE $__timeFilter(timestamp) GROUP BY time ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Datasource Health", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Trace Metadata Coverage", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 5, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 5, + "width": 18, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "observability", + "health", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Observability Health Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-operations.json b/devtools/grafana/dashboards/forge-operations.json new file mode 100644 index 00000000..960a0324 --- /dev/null +++ b/devtools/grafana/dashboards/forge-operations.json @@ -0,0 +1,921 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-operations", + "namespace": "default", + "uid": "forge-operations" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "API Up", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "up{job=\"forge-api\"}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Worker Up", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "up{job=\"forge-worker\"}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Webhook Events", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_webhooks_received_total[$__rate_interval])) by (source, event_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Webhook Failures", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_webhooks_failed_total[$__rate_interval])) by (source, error_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Workflow Starts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_workflows_started_total{ticket_type=~\"$ticket_type\"}[$__rate_interval])) by (ticket_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Agent Invocations", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(rate(forge_agent_invocations_total[$__rate_interval])) by (task_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Jira Stream Length", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "XLEN forge:events:jira", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "GitHub Stream Length", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "XLEN forge:events:github", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Retry Queue Depth", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "ZCARD forge:retry:queue", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Dead Letter Depth", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "redis-datasource", + "version": "v0", + "datasource": { + "name": "forge-redis" + }, + "spec": { + "query": "LLEN forge:retry:dlq", + "type": "cli" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Service Health", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 0, + "width": 6, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Runtime Throughput", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Queues", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 5, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "operations", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Operations Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-ticket-execution.json b/devtools/grafana/dashboards/forge-ticket-execution.json new file mode 100644 index 00000000..8985c54c --- /dev/null +++ b/devtools/grafana/dashboards/forge-ticket-execution.json @@ -0,0 +1,592 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-ticket-execution", + "namespace": "default", + "uid": "forge-ticket-execution" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Trace Timeline", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.timestamp as time, t.session_id as issue, t.metadata['workflow_step'] as step, any(o.provided_model_name) as model, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL LEFT JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) GROUP BY time, issue, step ORDER BY time" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Cost by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY cost DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Tokens by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY input_tokens DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Latency by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as latency_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) AND t.metadata['workflow_step'] != '' GROUP BY t.id, step ORDER BY latency_s DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Agent Calls", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.timestamp as time, t.session_id as issue, t.metadata['workflow_step'] as step, o.name as observation, o.provided_model_name as model, o.usage_details['input'] as input_tokens, o.usage_details['output'] as output_tokens, round(o.total_cost, 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.session_id IN ($jira_issue) ORDER BY time DESC LIMIT 200" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Ticket Timeline", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Step Breakdown", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Agent Calls", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "ticket", + "debug", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Ticket Execution Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/dashboards/forge-workflow-funnel.json b/devtools/grafana/dashboards/forge-workflow-funnel.json new file mode 100644 index 00000000..1524f1da --- /dev/null +++ b/devtools/grafana/dashboards/forge-workflow-funnel.json @@ -0,0 +1,654 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v2beta1", + "metadata": { + "name": "forge-workflow-funnel", + "namespace": "default", + "uid": "forge-workflow-funnel" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Workflow Starts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_started_total{ticket_type=~\"$ticket_type\"}) by (ticket_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Workflow Completions", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_completed_total{ticket_type=~\"$ticket_type\"}) by (ticket_type, final_node)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Workflow Failures", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "forge-prometheus" + }, + "spec": { + "expr": "sum(forge_workflows_failed_total{ticket_type=~\"$ticket_type\"}) by (ticket_type, error_type)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Trace Count by Step", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT metadata['workflow_step'] as step, uniqExact(session_id) as issues, count() as traces FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND metadata['workflow_step'] IN ($workflow_step) AND metadata['ticket_type'] IN ($ticket_type) GROUP BY step ORDER BY traces DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Average Step Duration", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT t.metadata['workflow_step'] as step, round(avg(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))), 2) as avg_latency_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE $__timeFilter(t.timestamp) AND t.metadata['workflow_step'] IN ($workflow_step) GROUP BY step ORDER BY avg_latency_s DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Rework Loops", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT session_id as issue, metadata['workflow_step'] as step, count() as trace_count FROM default.traces FINAL WHERE $__timeFilter(timestamp) AND session_id IN ($jira_issue) AND metadata['workflow_step'] != '' GROUP BY issue, step HAVING trace_count > 1 ORDER BY trace_count DESC LIMIT 50" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Workflow Outcomes", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Step Funnel", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Rework", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "forge", + "workflow", + "experimental" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "title": "Forge Workflow Funnel Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "project_id", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "label": "Project", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "ticket_type", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Ticket Type", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['ticket_type'] as tt FROM default.traces FINAL WHERE tt != '' ORDER BY tt", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "jira_issue", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "JIRA Issue", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "workflow_step", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Workflow Step", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "__legacyStringValue": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "SELECT DISTINCT metadata['workflow_step'] as step FROM default.traces FINAL WHERE step != '' ORDER BY step", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + } + ] + } +} diff --git a/devtools/grafana/provisioning/dashboards/dashboards.yml b/devtools/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..0eb9b95b --- /dev/null +++ b/devtools/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: default + type: file + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/devtools/grafana/provisioning/datasources/clickhouse.yml b/devtools/grafana/provisioning/datasources/clickhouse.yml new file mode 100644 index 00000000..056a7827 --- /dev/null +++ b/devtools/grafana/provisioning/datasources/clickhouse.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: ClickHouse + uid: langfuse-clickhouse + type: grafana-clickhouse-datasource + access: proxy + jsonData: + host: ${CLICKHOUSE_HOST} + port: ${CLICKHOUSE_PORT} + protocol: native + username: ${CLICKHOUSE_USER} + defaultDatabase: ${CLICKHOUSE_DATABASE} + secureJsonData: + password: ${CLICKHOUSE_PASSWORD} diff --git a/devtools/grafana/provisioning/datasources/prometheus.yml b/devtools/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..a118112a --- /dev/null +++ b/devtools/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: forge-prometheus + type: prometheus + access: proxy + url: http://${PROMETHEUS_HOST}:${PROMETHEUS_PORT} + isDefault: false + jsonData: + httpMethod: POST + timeInterval: 15s diff --git a/devtools/grafana/provisioning/datasources/redis.yml b/devtools/grafana/provisioning/datasources/redis.yml new file mode 100644 index 00000000..906d2566 --- /dev/null +++ b/devtools/grafana/provisioning/datasources/redis.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Redis + uid: forge-redis + type: redis-datasource + access: proxy + url: redis://${REDIS_HOST}:${REDIS_PORT} + isDefault: false diff --git a/docker-compose.yml b/docker-compose.yml index 4d20635a..fe882d95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,9 +49,36 @@ services: - '--storage.tsdb.path=/prometheus' - '--storage.tsdb.retention.time=15d' - '--web.enable-lifecycle' + + # Grafana dashboards for Forge observability + grafana: + image: grafana/grafana:latest + ports: + - "${GRAFANA_PORT:-3010}:3000" + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-grafana} + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource,redis-datasource + - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-clickhouse} + - CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-9000} + - CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-default} + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-clickhouse} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse} + - PROMETHEUS_HOST=prometheus + - PROMETHEUS_PORT=9090 + - REDIS_HOST=redis + - REDIS_PORT=6379 + volumes: + - grafana-data:/var/lib/grafana + - ./devtools/grafana/provisioning:/etc/grafana/provisioning:ro,z + - ./devtools/grafana/dashboards:/var/lib/grafana/dashboards:ro,z depends_on: - - forge-api + - prometheus + - redis + extra_hosts: + - "host.containers.internal:host-gateway" volumes: redis-data: prometheus-data: + grafana-data: diff --git a/docs/developer-guide.md b/docs/developer-guide.md index c4d8f79a..8259cdb3 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -175,7 +175,7 @@ REDIS_URL=redis://localhost:6380/0 # matches docker-compose port mapping ```bash CONTAINER_IMAGE=localhost/forge-dev:latest # built with podman above -CONTAINER_TIMEOUT=7200 # 2 hours max +CONTAINER_TIMEOUT=1800 # 30 minutes max CONTAINER_MEMORY=4g CONTAINER_CPUS=2 ``` @@ -188,6 +188,17 @@ LANGFUSE_ENABLED=true LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... LANGFUSE_HOST=https://cloud.langfuse.com +LANGFUSE_TRACE_TAGS=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,llm_model +LANGFUSE_TRACE_METADATA=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,retry_count,system_prompt_length,llm_model + +# Grafana dashboard stack +GRAFANA_PORT=3010 +LANGFUSE_DOCKER_NETWORK=langfuse_default +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_PORT=9000 +CLICKHOUSE_DATABASE=default +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_PASSWORD=clickhouse # CI behaviour CI_FIX_MAX_RETRIES=5 @@ -226,18 +237,30 @@ uv run forge worker > **Why not in Docker?** The worker spawns Podman containers. Running it inside Docker would require socket mounting which is not supported in this setup. -### Start Prometheus (optional, for metrics) +### Start Prometheus and Grafana (optional, for observability) ```bash -docker compose up prometheus -d -# Dashboard at http://localhost:9092 +docker compose --env-file .env -f docker-compose.yml up -d prometheus grafana +# Prometheus at http://localhost:9092 +# Grafana dashboards at http://localhost:3010 ``` +**With self-hosted Langfuse** — add the network override so Grafana can reach ClickHouse: + +```bash +docker compose --env-file .env \ + -f docker-compose.yml \ + -f devtools/grafana/compose.langfuse-network.yml \ + up -d prometheus grafana +``` + +> `compose.langfuse-network.yml` requires the `langfuse_default` Docker network to already exist. Omit it if you are not running self-hosted Langfuse — the Prometheus and Redis datasources work without it. + ### Full local stack ```bash -# Terminal 1 — Redis (and optionally Prometheus) -docker compose up redis prometheus -d +# Terminal 1 — Redis, Prometheus, and Grafana +docker compose --env-file .env -f docker-compose.yml up -d redis prometheus grafana # Terminal 2 — API server uv run uvicorn forge.main:app --reload --port 8000 --host 0.0.0.0 @@ -458,6 +481,7 @@ curl -X POST http://localhost:8000/api/v1/webhooks/github \ | API server | `http://localhost:8000/metrics` | | Worker | `http://localhost:8001/metrics` | | Prometheus UI | `http://localhost:9092` | +| Grafana dashboards | `http://localhost:3010` | ### Key metrics to watch @@ -556,6 +580,41 @@ Then set `LANGFUSE_HOST=http://localhost:3000` in `.env`. - CI fix analysis and implementation calls - All calls have the ticket key, task type, and token counts attached +### Trace tags and metadata + +You can attach workflow context to every Langfuse trace as tags or metadata, making it easy to filter traces by ticket type, CI status, event source, and so on. + +Configure the fields you want via two comma-separated env vars: + +```bash +# Fields to attach as searchable Langfuse tags. +# Keep these enabled for Grafana dashboards. +LANGFUSE_TRACE_TAGS=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,llm_model + +# Fields to attach as Langfuse trace metadata. +# Grafana ClickHouse queries require project_id, ticket_type, workflow_step, and ticket_key. +LANGFUSE_TRACE_METADATA=ticket_key,ticket_type,project_id,workflow_step,repo,pr_number,ci_status,event_source,event_type,retry_count,system_prompt_length,llm_model +``` + +Available field names for both settings: + +| Field | Description | +|-------|-------------| +| `ticket_key` | Jira ticket key (e.g. `PROJ-123`) | +| `ticket_type` | Ticket type (`Feature`, `Bug`, etc.) | +| `project_id` | Jira project ID derived from the ticket key | +| `workflow_step` | Name of the workflow node that triggered the call | +| `repo` | GitHub repository (`owner/repo`) | +| `pr_number` | Pull request number | +| `ci_status` | Last known CI status | +| `event_type` | Webhook event type that started the workflow | +| `event_source` | Source system (e.g. `jira`, `github`) | +| `retry_count` | Number of retries so far | +| `system_prompt_length` | Length of the system prompt in characters (metadata only) | +| `llm_model` | Model used for the call | + +Invalid or unknown field names are ignored and logged at `WARNING` level on startup. + ### Disabling tracing ```bash @@ -770,6 +829,7 @@ curl -X POST http://localhost:8000/api/v1/webhooks/github \ | Worker metrics | 8001 | `http://localhost:8001/metrics` | | Redis | 6380 | `redis://localhost:6380/0` | | Prometheus | 9092 | `http://localhost:9092` | +| Grafana | 3010 | `http://localhost:3010` | ### API endpoints diff --git a/docs/reference/config.md b/docs/reference/config.md index cc6ffebe..72f94b5d 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -102,6 +102,28 @@ Use these to skip the Jira project property requirement during local development | `LANGFUSE_PUBLIC_KEY` | Langfuse public key | | `LANGFUSE_SECRET_KEY` | Langfuse secret key | | `LANGFUSE_HOST` | Langfuse host (defaults to cloud; set for self-hosted) | +| `LANGFUSE_TRACE_TAGS` | Comma-separated list of trace attributes to attach as Langfuse tags. Available values: `ticket_key`, `ticket_type`, `project_id`, `workflow_step`, `repo`, `pr_number`, `ci_status`, `event_source`, `event_type`, `llm_model`. Default: empty (no tags). | +| `LANGFUSE_TRACE_METADATA` | Comma-separated list of trace attributes to attach as Langfuse metadata. Available values: same as tags plus `retry_count`, `system_prompt_length`. Default: empty (no metadata). | + +### Grafana Dashboards + +These variables are used by `docker-compose.yml`, `devtools/docker-compose.dev.yml`, and `devtools/grafana/compose.grafana.yml`. + +| Variable | Description | +|----------|-------------| +| `GRAFANA_PORT` | Host port for Grafana (default: `3010`) | +| `GRAFANA_ADMIN_USER` | Grafana admin user (default: `admin`) | +| `GRAFANA_ADMIN_PASSWORD` | Grafana admin password (default: `grafana`) | +| `LANGFUSE_DOCKER_NETWORK` | External Docker/Podman network for self-hosted Langfuse when using `devtools/grafana/compose.langfuse-network.yml` (default: `langfuse_default`) | +| `CLICKHOUSE_HOST` | Langfuse ClickHouse host reachable from the Grafana container | +| `CLICKHOUSE_PORT` | Langfuse ClickHouse native protocol port (default: `9000`) | +| `CLICKHOUSE_DATABASE` | Langfuse ClickHouse database (default: `default`) | +| `CLICKHOUSE_USER` | Langfuse ClickHouse user | +| `CLICKHOUSE_PASSWORD` | Langfuse ClickHouse password | +| `PROMETHEUS_HOST` | Prometheus host for standalone Grafana compose | +| `PROMETHEUS_PORT` | Prometheus port for standalone Grafana compose | +| `REDIS_HOST` | Redis host for standalone Grafana compose | +| `REDIS_PORT` | Redis port for standalone Grafana compose | ### MCP Servers diff --git a/src/forge/config.py b/src/forge/config.py index 0a10ea17..26721644 100644 --- a/src/forge/config.py +++ b/src/forge/config.py @@ -1,11 +1,17 @@ """Configuration management using Pydantic settings.""" -from functools import lru_cache -from typing import Literal +import logging +from functools import cached_property, lru_cache +from typing import TYPE_CHECKING, Literal from pydantic import Field, SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict +if TYPE_CHECKING: + from forge.integrations.langfuse.fields import TracingField + +logger = logging.getLogger(__name__) + class Settings(BaseSettings): """Application settings loaded from environment variables.""" @@ -203,6 +209,14 @@ def detect_model_provider(model_name: str) -> str: langfuse_host: str = Field( default="https://cloud.langfuse.com", description="Langfuse host URL" ) + langfuse_trace_tags: str = Field( + default="", + description="Comma-separated list of TracingField names to include as Langfuse trace tags", + ) + langfuse_trace_metadata: str = Field( + default="", + description="Comma-separated list of TracingField names to include as Langfuse trace metadata", + ) # Claude Agent SDK Configuration agent_enable_tools: bool = Field( @@ -351,6 +365,32 @@ def langfuse_enabled(self) -> bool: and self.langfuse_secret_key.get_secret_value() ) + @cached_property + def trace_tag_fields(self) -> list["TracingField"]: + """Parse and validate configured Langfuse trace tag fields.""" + from forge.integrations.langfuse.fields import parse_trace_fields + + fields = parse_trace_fields(self.langfuse_trace_tags, allow_tags=True) + if fields: + logger.info( + "Langfuse trace tags configured: %s", + ", ".join(f.value for f in fields), + ) + return fields + + @cached_property + def trace_metadata_fields(self) -> list["TracingField"]: + """Parse and validate configured Langfuse trace metadata fields.""" + from forge.integrations.langfuse.fields import parse_trace_fields + + fields = parse_trace_fields(self.langfuse_trace_metadata, allow_tags=False) + if fields: + logger.info( + "Langfuse trace metadata configured: %s", + ", ".join(f.value for f in fields), + ) + return fields + @property def use_vertex_ai(self) -> bool: """Check if using Vertex AI instead of direct Anthropic API.""" diff --git a/src/forge/integrations/agents/agent.py b/src/forge/integrations/agents/agent.py index f9a55ee5..ea4986a7 100644 --- a/src/forge/integrations/agents/agent.py +++ b/src/forge/integrations/agents/agent.py @@ -32,6 +32,7 @@ from forge.config import Settings, get_settings from forge.integrations.langfuse import get_langfuse_config, get_langfuse_context +from forge.integrations.langfuse.fields import resolve_trace_fields from forge.prompts import load_prompt, set_default_version from forge.skills.resolver import resolve_skill_paths @@ -57,6 +58,44 @@ logger = logging.getLogger(__name__) +_TRACE_FIELD_KEYS = frozenset( + { + "ticket_key", + "ticket_type", + "current_node", + "current_repo", + "current_pr_number", + "ci_status", + "event_type", + "event_source", + "retry_count", + "repo", + "pr_number", + } +) + + +def _forward_trace_fields(context: dict[str, Any] | None) -> dict[str, Any]: + """Extract trace-relevant fields from an incoming context dict.""" + if not context: + return {} + return {k: v for k, v in context.items() if k in _TRACE_FIELD_KEYS} + + +def _prompt_context_fields( + context: dict[str, Any] | None, allowed_keys: tuple[str, ...] +) -> dict[str, Any] | str: + """Extract task-relevant fields for rendered prompts, excluding trace-only data.""" + if not context: + return "None provided" + + prompt_context = { + key: context[key] + for key in allowed_keys + if key in context and context[key] not in (None, "") + } + return prompt_context or "None provided" + def get_weather(city: str) -> str: """Placeholder tool for agent testing.""" @@ -532,6 +571,8 @@ async def _run_agent( session_id: str | None = None, trace_name: str | None = None, ticket_key: str | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, ) -> str: """Run the agent with the given prompt. @@ -544,6 +585,8 @@ async def _run_agent( session_id: Optional session ID for Langfuse (e.g., ticket key). trace_name: Optional trace name for Langfuse. ticket_key: Optional ticket key for per-project skill resolution. + tags: Optional list of trace tags for Langfuse. + metadata: Optional metadata dict for Langfuse. Returns: Agent response text. @@ -565,7 +608,8 @@ async def _run_agent( langfuse_config = get_langfuse_config( trace_name=trace_name or "deep_agent_invocation", session_id=session_id, - metadata={"system_prompt_length": str(len(system_prompt))}, + tags=tags, + metadata=metadata, ) if langfuse_config: # Extract context params and remove from config @@ -580,6 +624,7 @@ async def _run_agent( session_id=langfuse_ctx_params.get("session_id"), user_id=langfuse_ctx_params.get("user_id"), tags=langfuse_ctx_params.get("tags"), + metadata=langfuse_ctx_params.get("metadata"), ): last_error: Exception | None = None for attempt in range(self.MAX_RETRIES): @@ -664,6 +709,7 @@ async def run_task( task: str, prompt: str, context: dict[str, Any] | None = None, + trace_context: dict[str, Any] | None = None, include_tools: bool = True, ) -> str: """Run a task, letting the agent choose the best approach. @@ -674,7 +720,9 @@ async def run_task( Args: task: Short task name for logging (e.g., 'generate-prd'). prompt: The task description and content to process. - context: Optional context variables for the prompt. + context: Optional context variables injected into the system prompt. + trace_context: Optional workflow fields forwarded to Langfuse only — + not written to the system prompt. include_tools: Whether to include MCP tools. Default True. Returns: @@ -694,8 +742,11 @@ async def run_task( for key, value in context.items(): system_prompt += f"- {key}: {value}\n" - # Extract ticket key for session tracking + # Extract ticket key for session tracking. Prefer prompt context for + # backward compatibility, but allow trace-only context to carry it. ticket_key = context.get("ticket_key") if context else None + if ticket_key is None and trace_context: + ticket_key = trace_context.get("ticket_key") import time @@ -704,6 +755,16 @@ async def run_task( logger.info(f"Running task '{task}' using Deep Agents") record_agent_invocation(task_type=task) _start = time.monotonic() + # Merge prompt context + trace-only fields for Langfuse resolution. + # trace_context fields are intentionally excluded from system_prompt above. + trace_state: dict[str, Any] = { + **(context or {}), + **(trace_context or {}), + "system_prompt_length": len(system_prompt), + "llm_model": self.settings.claude_model, + } + trace_tags, trace_metadata = resolve_trace_fields(trace_state) + result = await self._run_agent( prompt=prompt, system_prompt=system_prompt, @@ -711,6 +772,8 @@ async def run_task( session_id=ticket_key, trace_name=f"task:{task}", ticket_key=ticket_key, + tags=trace_tags or None, + metadata=trace_metadata or None, ) observe_agent_duration(task_type=task, duration=time.monotonic() - _start) @@ -861,7 +924,7 @@ async def generate_prd( prompt = load_prompt( "generate-prd", raw_requirements=raw_requirements, - context=context or "None provided", + context=_prompt_context_fields(context, ("project_key", "summary")), ) logger.info("Generating PRD using Deep Agents with skill") @@ -872,6 +935,7 @@ async def generate_prd( "ticket_key": context.get("ticket_key", "") if context else "", "project_key": context.get("project_key", "") if context else "", }, + trace_context=_forward_trace_fields(context), ) result = self._strip_preamble(result) @@ -897,7 +961,7 @@ async def generate_spec( prompt = load_prompt( "generate-spec", prd_content=prd_content, - context=context or "None provided", + context=_prompt_context_fields(context, ("project_key", "summary")), ) logger.info("Generating Spec using Deep Agents with skill") @@ -908,6 +972,7 @@ async def generate_spec( "ticket_key": context.get("ticket_key", "") if context else "", "project_key": context.get("project_key", "") if context else "", }, + trace_context=_forward_trace_fields(context), ) result = self._strip_preamble(result) @@ -963,6 +1028,7 @@ async def generate_epics( "feature_summary": context.get("feature_summary", "") if context else "", "available_repos": available_repos, }, + trace_context=_forward_trace_fields(context), ) epics = self._parse_epics_response(result) @@ -975,6 +1041,7 @@ async def regenerate_with_feedback( feedback: str, content_type: str, ticket_key: str | None = None, + context: dict[str, Any] | None = None, ) -> str: """Regenerate content incorporating feedback. @@ -982,6 +1049,8 @@ async def regenerate_with_feedback( original_content: The original generated content. feedback: User feedback/revision request. content_type: Type of content (prd, spec, epic). + ticket_key: Optional ticket key for session tracking. + context: Optional context with trace fields from workflow state. Returns: Regenerated content. @@ -1006,6 +1075,7 @@ async def regenerate_with_feedback( task=skill_name, prompt=prompt, context={"is_revision": True, "ticket_key": ticket_key or ""}, + trace_context=_forward_trace_fields(context), ) result = self._strip_preamble(result) @@ -1097,6 +1167,7 @@ async def answer_question( "artifact_type": artifact_type, "ticket_key": context.get("ticket_key", ""), }, + trace_context=_forward_trace_fields(context), # Q&A gets read-only MCP tools for lookups (filtered by agent_mcp_read_only) ) diff --git a/src/forge/integrations/langfuse/__init__.py b/src/forge/integrations/langfuse/__init__.py index 5586bcd9..b8f39d31 100644 --- a/src/forge/integrations/langfuse/__init__.py +++ b/src/forge/integrations/langfuse/__init__.py @@ -1,5 +1,9 @@ """Langfuse integration for LLM observability.""" +from forge.integrations.langfuse.fields import ( + TracingField, + resolve_trace_fields, +) from forge.integrations.langfuse.tracing import ( get_langfuse_config, get_langfuse_context, @@ -9,9 +13,11 @@ ) __all__ = [ + "TracingField", "get_langfuse_config", "get_langfuse_context", "get_langfuse_handler", + "resolve_trace_fields", "shutdown_langfuse", "trace_llm_call", ] diff --git a/src/forge/integrations/langfuse/fields.py b/src/forge/integrations/langfuse/fields.py new file mode 100644 index 00000000..fe121a30 --- /dev/null +++ b/src/forge/integrations/langfuse/fields.py @@ -0,0 +1,213 @@ +"""Configurable Langfuse trace tag and metadata fields. + +Admins configure which fields to include as tags/metadata via env vars: + LANGFUSE_TRACE_TAGS=ticket_type,project_id,workflow_step + LANGFUSE_TRACE_METADATA=ticket_key,ticket_type,project_id,retry_count +""" + +import logging +from enum import StrEnum +from typing import Any + +logger = logging.getLogger(__name__) + + +class TracingField(StrEnum): + """Available fields for Langfuse trace tags and metadata.""" + + TICKET_KEY = "ticket_key" + TICKET_TYPE = "ticket_type" + PROJECT_ID = "project_id" + WORKFLOW_STEP = "workflow_step" + REPO = "repo" + PR_NUMBER = "pr_number" + CI_STATUS = "ci_status" + EVENT_SOURCE = "event_source" + EVENT_TYPE = "event_type" + RETRY_COUNT = "retry_count" + SYSTEM_PROMPT_LENGTH = "system_prompt_length" + LLM_MODEL = "llm_model" + + @property + def tag_eligible(self) -> bool: + return self not in _METADATA_ONLY_FIELDS + + +_METADATA_ONLY_FIELDS = frozenset({TracingField.RETRY_COUNT, TracingField.SYSTEM_PROMPT_LENGTH}) + + +def resolve_field(field: TracingField, state: dict[str, Any]) -> str | None: + """Resolve a single tracing field from workflow state. + + Args: + field: The field to resolve. + state: Workflow state dict. + + Returns: + String value or None if the data isn't available. + """ + resolver = _RESOLVERS.get(field) + if resolver is None: + return None + return resolver(state) + + +def _resolve_ticket_key(state: dict[str, Any]) -> str | None: + val = state.get("ticket_key") + return str(val) if val is not None else None + + +def _resolve_ticket_type(state: dict[str, Any]) -> str | None: + val = state.get("ticket_type") + return str(val) if val is not None else None + + +def _resolve_project_id(state: dict[str, Any]) -> str | None: + ticket_key = state.get("ticket_key") + if not ticket_key or "-" not in str(ticket_key): + return None + return str(ticket_key).rsplit("-", 1)[0] + + +def _resolve_workflow_step(state: dict[str, Any]) -> str | None: + val = state.get("current_node") + return str(val) if val is not None else None + + +def _resolve_repo(state: dict[str, Any]) -> str | None: + val = state.get("repo") or state.get("current_repo") + return str(val) if val is not None else None + + +def _resolve_pr_number(state: dict[str, Any]) -> str | None: + val = state.get("pr_number") or state.get("current_pr_number") + return str(val) if val is not None else None + + +def _resolve_ci_status(state: dict[str, Any]) -> str | None: + val = state.get("ci_status") + return str(val) if val is not None else None + + +def _resolve_event_source(state: dict[str, Any]) -> str | None: + val = state.get("event_source") + if val is not None: + return str(val) + ctx = state.get("context") + if isinstance(ctx, dict): + val = ctx.get("source") + if val is not None: + return str(val) + return None + + +def _resolve_event_type(state: dict[str, Any]) -> str | None: + val = state.get("event_type") + return str(val) if val is not None else None + + +def _resolve_retry_count(state: dict[str, Any]) -> str | None: + val = state.get("retry_count") + return str(val) if val is not None else None + + +def _resolve_system_prompt_length(state: dict[str, Any]) -> str | None: + val = state.get("system_prompt_length") + return str(val) if val is not None else None + + +def _resolve_llm_model(state: dict[str, Any]) -> str | None: + val = state.get("llm_model") + return str(val) if val is not None else None + + +_RESOLVERS: dict[TracingField, Any] = { + TracingField.TICKET_KEY: _resolve_ticket_key, + TracingField.TICKET_TYPE: _resolve_ticket_type, + TracingField.PROJECT_ID: _resolve_project_id, + TracingField.WORKFLOW_STEP: _resolve_workflow_step, + TracingField.REPO: _resolve_repo, + TracingField.PR_NUMBER: _resolve_pr_number, + TracingField.CI_STATUS: _resolve_ci_status, + TracingField.EVENT_SOURCE: _resolve_event_source, + TracingField.EVENT_TYPE: _resolve_event_type, + TracingField.RETRY_COUNT: _resolve_retry_count, + TracingField.SYSTEM_PROMPT_LENGTH: _resolve_system_prompt_length, + TracingField.LLM_MODEL: _resolve_llm_model, +} + + +def parse_trace_fields(config_str: str, *, allow_tags: bool) -> list[TracingField]: + """Parse a comma-separated config string into validated TracingField list. + + Args: + config_str: Comma-separated field names (e.g., "ticket_key,ticket_type"). + allow_tags: If True, validates tag eligibility; if False, allows all fields. + + Returns: + List of valid TracingField values. Invalid names are warned and skipped. + """ + if not config_str or not config_str.strip(): + return [] + + available = ", ".join(sorted(f.value for f in TracingField)) + result: list[TracingField] = [] + + for raw in config_str.split(","): + name = raw.strip() + if not name: + continue + + try: + field = TracingField(name) + except ValueError: + logger.warning( + "Invalid Langfuse trace field '%s' - not a recognized field name. Available: %s", + name, + available, + ) + continue + + if allow_tags and not field.tag_eligible: + logger.warning( + "Field '%s' is not eligible for tags (quantitative field) - skipping", + name, + ) + continue + + result.append(field) + + return result + + +def resolve_trace_fields(state: dict[str, Any]) -> tuple[list[str], dict[str, Any]]: + """Resolve configured tracing fields from workflow state. + + Reads the configured tag/metadata fields from settings, resolves each + against the state dict, and returns the results. Fields that resolve + to None are silently omitted. + + Args: + state: Workflow state dict containing trace data. + + Returns: + (tags, metadata) — tags is a list of raw string values, + metadata is a dict of field_name -> string value. + """ + from forge.config import get_settings + + settings = get_settings() + + tags: list[str] = [] + for field in settings.trace_tag_fields: + value = resolve_field(field, state) + if value: + tags.append(value) + + metadata: dict[str, Any] = {} + for field in settings.trace_metadata_fields: + value = resolve_field(field, state) + if value is not None: + metadata[field.value] = value + + return tags, metadata diff --git a/src/forge/integrations/langfuse/tracing.py b/src/forge/integrations/langfuse/tracing.py index d514a4ce..9b3086e5 100644 --- a/src/forge/integrations/langfuse/tracing.py +++ b/src/forge/integrations/langfuse/tracing.py @@ -210,6 +210,7 @@ def get_langfuse_config( "session_id": session_id, "user_id": user_id, "tags": tags, + "metadata": metadata, } return config diff --git a/src/forge/workflow/base.py b/src/forge/workflow/base.py index f99945c2..b3b0d161 100644 --- a/src/forge/workflow/base.py +++ b/src/forge/workflow/base.py @@ -17,6 +17,9 @@ class BaseState(TypedDict, total=False): thread_id: str ticket_key: str + # Event origin + event_type: str + # Execution control current_node: str is_paused: bool diff --git a/src/forge/workflow/nodes/code_review.py b/src/forge/workflow/nodes/code_review.py index b3f4e9ac..95fb7692 100644 --- a/src/forge/workflow/nodes/code_review.py +++ b/src/forge/workflow/nodes/code_review.py @@ -146,6 +146,15 @@ async def sync_pr_description( task="sync-pr-description", prompt=prompt, context={"owner": owner, "repo": repo, "pr_number": pr_number}, + trace_context={ + "ticket_key": state.get("ticket_key", ""), + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "ci_status": state.get("ci_status", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, include_tools=False, ) finally: diff --git a/src/forge/workflow/nodes/epic_decomposition.py b/src/forge/workflow/nodes/epic_decomposition.py index 6b6e2547..036bad5b 100644 --- a/src/forge/workflow/nodes/epic_decomposition.py +++ b/src/forge/workflow/nodes/epic_decomposition.py @@ -114,6 +114,11 @@ async def decompose_epics(state: WorkflowState) -> WorkflowState: # Build context for Epic generation context: dict[str, Any] = { "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), "project_key": project_key, "feature_summary": parent_issue.summary, "available_repos": available_repos, @@ -319,6 +324,13 @@ async def update_single_epic(state: WorkflowState) -> WorkflowState: feedback=feedback, content_type="epic", ticket_key=ticket_key, + context={ + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, ) # Update Epic description diff --git a/src/forge/workflow/nodes/pr_creation.py b/src/forge/workflow/nodes/pr_creation.py index 294b5452..225bed8a 100644 --- a/src/forge/workflow/nodes/pr_creation.py +++ b/src/forge/workflow/nodes/pr_creation.py @@ -471,9 +471,19 @@ async def _generate_pr_body_with_agent( prompt=prompt, context={ "ticket_key": ticket_key, - "repo": current_repo, "task_count": len(implemented_tasks), }, + trace_context={ + "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "repo": current_repo, + "pr_number": state.get("current_pr_number", ""), + "ci_status": state.get("ci_status", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, include_tools=False, # No tools needed for text generation ) diff --git a/src/forge/workflow/nodes/prd_generation.py b/src/forge/workflow/nodes/prd_generation.py index ff929433..2b4a0529 100644 --- a/src/forge/workflow/nodes/prd_generation.py +++ b/src/forge/workflow/nodes/prd_generation.py @@ -194,6 +194,11 @@ async def generate_prd(state: WorkflowState) -> WorkflowState: # Build context from issue metadata context: dict[str, Any] = { "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), "summary": issue.summary, "project_key": issue.project_key, } @@ -306,6 +311,13 @@ async def regenerate_prd_with_feedback(state: WorkflowState) -> WorkflowState: feedback=feedback, content_type="prd", ticket_key=ticket_key, + context={ + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, ) # Publish revised PRD diff --git a/src/forge/workflow/nodes/qa_handler.py b/src/forge/workflow/nodes/qa_handler.py index ce1132ac..bcf961fc 100644 --- a/src/forge/workflow/nodes/qa_handler.py +++ b/src/forge/workflow/nodes/qa_handler.py @@ -75,9 +75,14 @@ async def answer_question(state: WorkflowState) -> WorkflowState: question=question, artifact_content=artifact_content, context={ + "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), "artifact_type": artifact_type, "generation_context": generation_context, - "ticket_key": ticket_key, }, ) diff --git a/src/forge/workflow/nodes/spec_generation.py b/src/forge/workflow/nodes/spec_generation.py index 876db47e..40b14583 100644 --- a/src/forge/workflow/nodes/spec_generation.py +++ b/src/forge/workflow/nodes/spec_generation.py @@ -69,6 +69,11 @@ async def generate_spec(state: WorkflowState) -> WorkflowState: # Build context context: dict[str, Any] = { "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), } # Generate specification using Claude - primary operation @@ -174,6 +179,13 @@ async def regenerate_spec_with_feedback(state: WorkflowState) -> WorkflowState: feedback=feedback, content_type="spec", ticket_key=ticket_key, + context={ + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, ) # Store updated spec in Jira (comment or custom field based on config) diff --git a/src/forge/workflow/nodes/task_generation.py b/src/forge/workflow/nodes/task_generation.py index cf211fb4..1435257c 100644 --- a/src/forge/workflow/nodes/task_generation.py +++ b/src/forge/workflow/nodes/task_generation.py @@ -106,6 +106,12 @@ async def generate_tasks(state: WorkflowState) -> WorkflowState: # Build context context: dict[str, Any] = { + "ticket_key": ticket_key, + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), "epic_key": epic_key, "epic_summary": epic_summary, "feature_key": ticket_key, @@ -519,6 +525,13 @@ async def update_single_task(state: WorkflowState) -> WorkflowState: feedback=feedback, content_type="task", ticket_key=ticket_key, + context={ + "ticket_type": state.get("ticket_type", ""), + "current_node": state.get("current_node", ""), + "event_type": state.get("event_type", ""), + "event_source": state.get("context", {}).get("source", ""), + "retry_count": state.get("retry_count", 0), + }, ) # Update Task in Jira diff --git a/tests/unit/devtools/test_grafana_assets.py b/tests/unit/devtools/test_grafana_assets.py new file mode 100644 index 00000000..6407f50c --- /dev/null +++ b/tests/unit/devtools/test_grafana_assets.py @@ -0,0 +1,63 @@ +"""Tests for checked-in Grafana dashboard assets.""" + +import json +from pathlib import Path + +GRAFANA_DIR = Path("devtools/grafana") +DASHBOARD_DIR = GRAFANA_DIR / "dashboards" +EXPECTED_DASHBOARDS = { + "forge-agent-performance.json", + "forge-business.json", + "forge-ci-review.json", + "forge-cost.json", + "forge-engineering.json", + "forge-issue-detail.json", + "forge-model-usage.json", + "forge-observability-health.json", + "forge-operations.json", + "forge-ticket-execution.json", + "forge-workflow-funnel.json", +} + + +def test_grafana_dashboard_json_is_valid() -> None: + dashboards = sorted(DASHBOARD_DIR.glob("*.json")) + assert dashboards, "expected checked-in Grafana dashboards" + assert {dashboard.name for dashboard in dashboards} == EXPECTED_DASHBOARDS + + for dashboard in dashboards: + data = json.loads(dashboard.read_text()) + assert data["kind"] == "Dashboard" + assert data["metadata"]["name"] == dashboard.stem + assert data["spec"]["title"] + assert data["spec"]["elements"], f"{dashboard} should define panels" + + +def test_dashboards_use_configurable_trace_fields() -> None: + joined = "\n".join(path.read_text() for path in sorted(DASHBOARD_DIR.glob("*.json"))) + + assert "OSASINFRA" not in joined + assert "OSPA" not in joined + assert "tags[1]" not in joined + assert "tags[3]" not in joined + assert "project_id=~" not in joined + assert "metadata['project_id']" in joined + assert "metadata['ticket_type']" in joined + assert "metadata['workflow_step']" in joined + + +def test_dashboards_cover_expected_datasources() -> None: + joined = "\n".join(path.read_text() for path in sorted(DASHBOARD_DIR.glob("*.json"))) + + assert "langfuse-clickhouse" in joined + assert "forge-prometheus" in joined + assert "forge-redis" in joined + + +def test_grafana_provisioning_files_exist() -> None: + assert (GRAFANA_DIR / "provisioning/dashboards/dashboards.yml").is_file() + assert (GRAFANA_DIR / "provisioning/datasources/clickhouse.yml").is_file() + assert (GRAFANA_DIR / "provisioning/datasources/prometheus.yml").is_file() + assert (GRAFANA_DIR / "provisioning/datasources/redis.yml").is_file() + assert (GRAFANA_DIR / "compose.grafana.yml").is_file() + assert (GRAFANA_DIR / "compose.langfuse-network.yml").is_file() diff --git a/tests/unit/integrations/agents/test_run_task_tracing.py b/tests/unit/integrations/agents/test_run_task_tracing.py new file mode 100644 index 00000000..ae1857bb --- /dev/null +++ b/tests/unit/integrations/agents/test_run_task_tracing.py @@ -0,0 +1,171 @@ +"""Tests for run_task() trace field resolution. + +Verifies that run_task() builds the trace_state correctly, calls +resolve_trace_fields(), and passes the resolved tags/metadata to +_run_agent(). +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from forge.integrations.agents.agent import ForgeAgent + + +@pytest.fixture +def agent() -> ForgeAgent: + return ForgeAgent() + + +def _metrics_patches(): + """Common patches for the inline-imported metrics helpers.""" + return ( + patch("forge.api.routes.metrics.record_agent_invocation"), + patch("forge.api.routes.metrics.observe_agent_duration"), + ) + + +class TestRunTaskTraceResolution: + """run_task() resolves trace fields and forwards them to _run_agent().""" + + @pytest.mark.asyncio + async def test_builds_trace_state_from_context_and_system_prompt( + self, agent: ForgeAgent + ) -> None: + context = {"ticket_key": "PROJ-42", "current_node": "generate_prd"} + + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields" + ) as mock_resolve, + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + mock_resolve.return_value = (["PROJ-42"], {"ticket_key": "PROJ-42"}) + + await agent.run_task(task="generate-prd", prompt="test", context=context) + + # resolve_trace_fields should receive merged state with system_prompt_length and llm_model + resolve_call_state = mock_resolve.call_args[0][0] + assert resolve_call_state["ticket_key"] == "PROJ-42" + assert resolve_call_state["current_node"] == "generate_prd" + assert "system_prompt_length" in resolve_call_state + assert isinstance(resolve_call_state["system_prompt_length"], int) + assert resolve_call_state["llm_model"] == agent.settings.claude_model + + @pytest.mark.asyncio + async def test_passes_resolved_tags_to_run_agent(self, agent: ForgeAgent) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields", + return_value=(["Bug", "PROJ"], {"ticket_key": "PROJ-42"}), + ), + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + await agent.run_task( + task="test-task", + prompt="test", + context={"ticket_key": "PROJ-42"}, + ) + + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["tags"] == ["Bug", "PROJ"] + assert call_kwargs["metadata"] == {"ticket_key": "PROJ-42"} + + @pytest.mark.asyncio + async def test_uses_trace_context_ticket_key_for_session_when_context_omits_it( + self, agent: ForgeAgent + ) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields", + return_value=([], {}), + ), + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + await agent.run_task( + task="test-task", + prompt="test", + context={"task_count": 2}, + trace_context={"ticket_key": "PROJ-42"}, + ) + + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["session_id"] == "PROJ-42" + assert call_kwargs["ticket_key"] == "PROJ-42" + assert "PROJ-42" not in call_kwargs["system_prompt"] + + @pytest.mark.asyncio + async def test_empty_tags_passed_as_none(self, agent: ForgeAgent) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields", + return_value=([], {}), + ), + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + await agent.run_task(task="test-task", prompt="test", context={}) + + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["tags"] is None + assert call_kwargs["metadata"] is None + + @pytest.mark.asyncio + async def test_none_context_produces_trace_state_with_prompt_and_model( + self, agent: ForgeAgent + ) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields" + ) as mock_resolve, + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + mock_resolve.return_value = ([], {}) + await agent.run_task(task="test-task", prompt="test", context=None) + + resolve_call_state = mock_resolve.call_args[0][0] + assert "system_prompt_length" in resolve_call_state + assert "llm_model" in resolve_call_state + # No other context keys should be present + assert len(resolve_call_state) == 2 + + @pytest.mark.asyncio + async def test_trace_name_uses_task_prefix(self, agent: ForgeAgent) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields", + return_value=([], {}), + ), + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + await agent.run_task(task="generate-prd", prompt="test") + + assert mock_run.call_args.kwargs["trace_name"] == "task:generate-prd" + + @pytest.mark.asyncio + async def test_session_id_from_ticket_key(self, agent: ForgeAgent) -> None: + with ( + patch.object(agent, "_run_agent", new_callable=AsyncMock) as mock_run, + patch( + "forge.integrations.agents.agent.resolve_trace_fields", + return_value=([], {}), + ), + patch("forge.integrations.agents.agent.load_prompt", return_value="prompt"), + ): + mock_run.return_value = "result" + await agent.run_task( + task="test", prompt="test", context={"ticket_key": "PROJ-42"} + ) + + assert mock_run.call_args.kwargs["session_id"] == "PROJ-42" diff --git a/tests/unit/integrations/agents/test_trace_forwarding.py b/tests/unit/integrations/agents/test_trace_forwarding.py new file mode 100644 index 00000000..f28764d5 --- /dev/null +++ b/tests/unit/integrations/agents/test_trace_forwarding.py @@ -0,0 +1,348 @@ +"""Tests for trace field forwarding in ForgeAgent. + +Covers _forward_trace_fields() utility and the agent methods that +pass trace context through to run_task(). +""" + +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from forge.integrations.agents.agent import ForgeAgent, _forward_trace_fields + + +class TestForwardTraceFields: + """Unit tests for _forward_trace_fields().""" + + def test_extracts_known_trace_keys(self) -> None: + context = { + "ticket_key": "PROJ-42", + "ticket_type": "Bug", + "current_node": "analyze_bug", + "current_repo": "acme/widgets", + "current_pr_number": 99, + "ci_status": "passed", + "event_type": "issue_updated", + "event_source": "jira", + "retry_count": 3, + "repo": "acme/widgets", + "pr_number": 55, + } + result = _forward_trace_fields(context) + assert result == context + + def test_filters_out_non_trace_keys(self) -> None: + context = { + "ticket_key": "PROJ-42", + "project_key": "PROJ", + "summary": "Fix the thing", + "workspace_path": "/tmp/ws", + "feature_summary": "A feature", + "available_repos": ["acme/widgets"], + } + result = _forward_trace_fields(context) + assert result == {"ticket_key": "PROJ-42"} + + def test_returns_empty_for_none(self) -> None: + assert _forward_trace_fields(None) == {} + + def test_returns_empty_for_empty_dict(self) -> None: + assert _forward_trace_fields({}) == {} + + def test_returns_empty_when_no_trace_keys_present(self) -> None: + context = {"summary": "Something", "workspace_path": "/tmp"} + assert _forward_trace_fields(context) == {} + + def test_preserves_original_values_unchanged(self) -> None: + context = { + "ticket_key": "PROJ-42", + "retry_count": 0, + "ci_status": "", + } + result = _forward_trace_fields(context) + assert result["ticket_key"] == "PROJ-42" + assert result["retry_count"] == 0 + assert result["ci_status"] == "" + + def test_does_not_mutate_input(self) -> None: + context = {"ticket_key": "PROJ-42", "summary": "Test"} + original = dict(context) + _forward_trace_fields(context) + assert context == original + + +class TestGeneratePrdTraceForwarding: + """generate_prd() uses _forward_trace_fields() and adds project_key.""" + + @pytest.mark.asyncio + async def test_forwards_trace_fields_to_run_task(self) -> None: + agent = ForgeAgent() + context = { + "ticket_key": "PROJ-42", + "ticket_type": "Feature", + "current_node": "generate_prd", + "event_type": "issue_updated", + "event_source": "jira", + "retry_count": 1, + "project_key": "PROJ", + "summary": "Build auth", + } + + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# PRD\n\nContent" + await agent.generate_prd("Build auth system", context=context) + + # Prompt context contains only task-relevant fields + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["ticket_key"] == "PROJ-42" + assert call_ctx["project_key"] == "PROJ" + assert "ticket_type" not in call_ctx + assert "summary" not in call_ctx + + # Trace fields forwarded to trace_context, not the prompt + call_trace = mock_run.call_args.kwargs["trace_context"] + assert call_trace["ticket_type"] == "Feature" + assert call_trace["current_node"] == "generate_prd" + assert call_trace["event_type"] == "issue_updated" + assert call_trace["event_source"] == "jira" + assert call_trace["retry_count"] == 1 + + rendered_prompt = mock_run.call_args.kwargs["prompt"] + assert "ticket_type" not in rendered_prompt + assert "current_node" not in rendered_prompt + assert "event_type" not in rendered_prompt + assert "event_source" not in rendered_prompt + assert "retry_count" not in rendered_prompt + + @pytest.mark.asyncio + async def test_handles_none_context(self) -> None: + agent = ForgeAgent() + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# PRD\n\nContent" + await agent.generate_prd("Build something", context=None) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx == {"ticket_key": "", "project_key": ""} + + @pytest.mark.asyncio + async def test_project_key_defaults_to_empty(self) -> None: + agent = ForgeAgent() + context: dict[str, Any] = {"ticket_key": "PROJ-42"} + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# PRD" + await agent.generate_prd("Requirements", context=context) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["project_key"] == "" + + +class TestGenerateSpecTraceForwarding: + """generate_spec() uses _forward_trace_fields() and adds project_key.""" + + @pytest.mark.asyncio + async def test_forwards_trace_fields(self) -> None: + agent = ForgeAgent() + context = { + "ticket_key": "PROJ-42", + "ticket_type": "Feature", + "current_node": "generate_spec", + "event_type": "issue_updated", + "project_key": "PROJ", + } + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# Spec\n\nContent" + await agent.generate_spec("PRD content", context=context) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["ticket_key"] == "PROJ-42" + assert call_ctx["project_key"] == "PROJ" + assert "ticket_type" not in call_ctx + + call_trace = mock_run.call_args.kwargs["trace_context"] + assert call_trace["ticket_type"] == "Feature" + assert call_trace["current_node"] == "generate_spec" + + rendered_prompt = mock_run.call_args.kwargs["prompt"] + assert "ticket_type" not in rendered_prompt + assert "current_node" not in rendered_prompt + assert "event_type" not in rendered_prompt + + +class TestGenerateEpicsTraceForwarding: + """generate_epics() uses _forward_trace_fields() and adds extra context.""" + + @pytest.mark.asyncio + async def test_forwards_trace_fields_plus_extra(self) -> None: + agent = ForgeAgent() + context = { + "ticket_key": "PROJ-42", + "ticket_type": "Feature", + "current_node": "decompose_epics", + "event_type": "issue_updated", + "event_source": "jira", + "retry_count": 0, + "project_key": "PROJ", + "feature_summary": "Auth system", + "available_repos": ["acme/backend", "acme/frontend"], + } + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "---\nEPIC: Test\nREPO: acme/backend\nPLAN:\n1. Do it\n---" + await agent.generate_epics("Spec content", context=context) + + # Prompt context contains only task-relevant fields + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["ticket_key"] == "PROJ-42" + assert call_ctx["project_key"] == "PROJ" + assert call_ctx["feature_summary"] == "Auth system" + assert call_ctx["available_repos"] == ["acme/backend", "acme/frontend"] + assert "ticket_type" not in call_ctx + assert "summary" not in call_ctx + + # Trace fields forwarded to trace_context only + call_trace = mock_run.call_args.kwargs["trace_context"] + assert call_trace["ticket_type"] == "Feature" + assert call_trace["current_node"] == "decompose_epics" + + +class TestRegenerateWithFeedbackTraceForwarding: + """regenerate_with_feedback() accepts context and uses _forward_trace_fields().""" + + @pytest.mark.asyncio + async def test_forwards_trace_fields_from_context(self) -> None: + agent = ForgeAgent() + context = { + "ticket_type": "Feature", + "current_node": "regenerate_prd", + "event_type": "issue_updated", + "event_source": "jira", + "retry_count": 2, + } + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# Revised PRD" + await agent.regenerate_with_feedback( + original_content="# Old PRD", + feedback="Add more detail", + content_type="prd", + ticket_key="PROJ-42", + context=context, + ) + + # Prompt context contains only the minimal fields needed for the task + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["is_revision"] is True + assert call_ctx["ticket_key"] == "PROJ-42" + assert "ticket_type" not in call_ctx + + # Trace fields forwarded to trace_context only + call_trace = mock_run.call_args.kwargs["trace_context"] + assert call_trace["ticket_type"] == "Feature" + assert call_trace["current_node"] == "regenerate_prd" + assert call_trace["event_type"] == "issue_updated" + assert call_trace["event_source"] == "jira" + assert call_trace["retry_count"] == 2 + + @pytest.mark.asyncio + async def test_handles_none_context(self) -> None: + agent = ForgeAgent() + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# Revised" + await agent.regenerate_with_feedback( + original_content="# Old", + feedback="Fix it", + content_type="spec", + ticket_key="PROJ-42", + context=None, + ) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx == {"is_revision": True, "ticket_key": "PROJ-42"} + + @pytest.mark.asyncio + async def test_ticket_key_defaults_to_empty(self) -> None: + agent = ForgeAgent() + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# Revised" + await agent.regenerate_with_feedback( + original_content="# Old", + feedback="Fix it", + content_type="prd", + ticket_key=None, + context=None, + ) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["ticket_key"] == "" + + @pytest.mark.asyncio + async def test_content_type_maps_to_correct_skill(self) -> None: + agent = ForgeAgent() + skill_map = { + "prd": "generate-prd", + "spec": "generate-spec", + "epic": "decompose-epics", + "task": "generate-tasks", + } + for content_type, expected_task in skill_map.items(): + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "# Result" + await agent.regenerate_with_feedback( + original_content="# Old", + feedback="Fix", + content_type=content_type, + ) + assert mock_run.call_args.kwargs["task"] == expected_task + + +class TestAnswerQuestionTraceForwarding: + """answer_question() uses _forward_trace_fields() for context.""" + + @pytest.mark.asyncio + async def test_forwards_trace_fields(self) -> None: + agent = ForgeAgent() + context = { + "ticket_key": "PROJ-42", + "ticket_type": "Feature", + "current_node": "answer_question", + "event_type": "issue_updated", + "event_source": "jira", + "retry_count": 0, + "artifact_type": "prd", + "generation_context": {"raw_requirements": "Build API"}, + } + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "The answer" + await agent.answer_question( + question="Why REST?", + artifact_content="# PRD\n\nWe use REST", + context=context, + ) + + # Prompt context contains only task-relevant fields + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["ticket_key"] == "PROJ-42" + assert call_ctx["artifact_type"] == "prd" + assert "ticket_type" not in call_ctx + assert "generation_context" not in call_ctx + + # Trace fields forwarded to trace_context only + call_trace = mock_run.call_args.kwargs["trace_context"] + assert call_trace["ticket_type"] == "Feature" + assert call_trace["current_node"] == "answer_question" + assert call_trace["event_type"] == "issue_updated" + assert call_trace["event_source"] == "jira" + assert call_trace["retry_count"] == 0 + + @pytest.mark.asyncio + async def test_artifact_type_defaults_to_document(self) -> None: + agent = ForgeAgent() + with patch.object(agent, "run_task", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "Answer" + await agent.answer_question( + question="What?", + artifact_content="Content", + context={}, + ) + + call_ctx = mock_run.call_args.kwargs["context"] + assert call_ctx["artifact_type"] == "document" diff --git a/tests/unit/integrations/langfuse/test_fields.py b/tests/unit/integrations/langfuse/test_fields.py new file mode 100644 index 00000000..6623e20e --- /dev/null +++ b/tests/unit/integrations/langfuse/test_fields.py @@ -0,0 +1,395 @@ +"""Tests for Langfuse tracing field resolvers.""" + +import logging +from typing import Any +from unittest.mock import PropertyMock, patch + +import pytest + +from forge.config import Settings +from forge.integrations.langfuse.fields import ( + TracingField, + parse_trace_fields, + resolve_field, + resolve_trace_fields, +) + + +def _make_state(**overrides: Any) -> dict[str, Any]: + """Build a minimal workflow state dict for testing.""" + base: dict[str, Any] = { + "ticket_key": "PROJ-42", + "ticket_type": "Bug", + "current_node": "analyze_bug", + "current_repo": "acme/widgets", + "current_pr_number": 99, + "ci_status": "passed", + "event_type": "issue_updated", + "retry_count": 3, + "context": {"source": "jira"}, + } + base.update(overrides) + return base + + +class TestFieldResolvers: + """Each TracingField resolver extracts the right value from state.""" + + def test_ticket_key(self) -> None: + assert resolve_field(TracingField.TICKET_KEY, _make_state()) == "PROJ-42" + + def test_ticket_key_missing(self) -> None: + state = _make_state() + del state["ticket_key"] + assert resolve_field(TracingField.TICKET_KEY, state) is None + + def test_ticket_type(self) -> None: + assert resolve_field(TracingField.TICKET_TYPE, _make_state()) == "Bug" + + def test_ticket_type_missing(self) -> None: + state = _make_state() + del state["ticket_type"] + assert resolve_field(TracingField.TICKET_TYPE, state) is None + + def test_project_id(self) -> None: + assert resolve_field(TracingField.PROJECT_ID, _make_state()) == "PROJ" + + def test_project_id_no_ticket_key(self) -> None: + state = _make_state() + del state["ticket_key"] + assert resolve_field(TracingField.PROJECT_ID, state) is None + + def test_project_id_no_dash(self) -> None: + assert resolve_field(TracingField.PROJECT_ID, _make_state(ticket_key="NODASH")) is None + + def test_workflow_step(self) -> None: + assert resolve_field(TracingField.WORKFLOW_STEP, _make_state()) == "analyze_bug" + + def test_workflow_step_missing(self) -> None: + state = _make_state() + del state["current_node"] + assert resolve_field(TracingField.WORKFLOW_STEP, state) is None + + def test_repo(self) -> None: + assert resolve_field(TracingField.REPO, _make_state()) == "acme/widgets" + + def test_repo_from_short_key(self) -> None: + state = _make_state() + del state["current_repo"] + state["repo"] = "acme/other" + assert resolve_field(TracingField.REPO, state) == "acme/other" + + def test_repo_prefers_short_key(self) -> None: + state = _make_state(repo="acme/preferred") + assert resolve_field(TracingField.REPO, state) == "acme/preferred" + + def test_repo_missing(self) -> None: + state = _make_state() + del state["current_repo"] + assert resolve_field(TracingField.REPO, state) is None + + def test_pr_number(self) -> None: + assert resolve_field(TracingField.PR_NUMBER, _make_state()) == "99" + + def test_pr_number_from_short_key(self) -> None: + state = _make_state() + del state["current_pr_number"] + state["pr_number"] = 42 + assert resolve_field(TracingField.PR_NUMBER, state) == "42" + + def test_pr_number_prefers_short_key(self) -> None: + state = _make_state(pr_number=42) + assert resolve_field(TracingField.PR_NUMBER, state) == "42" + + def test_pr_number_missing(self) -> None: + state = _make_state() + del state["current_pr_number"] + assert resolve_field(TracingField.PR_NUMBER, state) is None + + def test_ci_status(self) -> None: + assert resolve_field(TracingField.CI_STATUS, _make_state()) == "passed" + + def test_ci_status_missing(self) -> None: + state = _make_state() + del state["ci_status"] + assert resolve_field(TracingField.CI_STATUS, state) is None + + def test_event_source(self) -> None: + assert resolve_field(TracingField.EVENT_SOURCE, _make_state()) == "jira" + + def test_event_source_missing_context(self) -> None: + assert resolve_field(TracingField.EVENT_SOURCE, _make_state(context={})) is None + + def test_event_source_no_context_key(self) -> None: + state = _make_state() + del state["context"] + assert resolve_field(TracingField.EVENT_SOURCE, state) is None + + def test_event_type(self) -> None: + assert resolve_field(TracingField.EVENT_TYPE, _make_state()) == "issue_updated" + + def test_event_type_missing(self) -> None: + state = _make_state() + del state["event_type"] + assert resolve_field(TracingField.EVENT_TYPE, state) is None + + def test_retry_count(self) -> None: + assert resolve_field(TracingField.RETRY_COUNT, _make_state()) == "3" + + def test_retry_count_missing(self) -> None: + state = _make_state() + del state["retry_count"] + assert resolve_field(TracingField.RETRY_COUNT, state) is None + + def test_retry_count_zero(self) -> None: + assert resolve_field(TracingField.RETRY_COUNT, _make_state(retry_count=0)) == "0" + + def test_system_prompt_length(self) -> None: + state = _make_state(system_prompt_length=4523) + assert resolve_field(TracingField.SYSTEM_PROMPT_LENGTH, state) == "4523" + + def test_system_prompt_length_missing(self) -> None: + assert resolve_field(TracingField.SYSTEM_PROMPT_LENGTH, _make_state()) is None + + def test_llm_model(self) -> None: + state = _make_state(llm_model="claude-sonnet-4-6-20250514") + assert resolve_field(TracingField.LLM_MODEL, state) == "claude-sonnet-4-6-20250514" + + def test_llm_model_missing(self) -> None: + assert resolve_field(TracingField.LLM_MODEL, _make_state()) is None + + +class TestTagEligibility: + """Verify which fields are tag-eligible vs metadata-only.""" + + @pytest.mark.parametrize( + "field", + [ + TracingField.TICKET_KEY, + TracingField.TICKET_TYPE, + TracingField.PROJECT_ID, + TracingField.WORKFLOW_STEP, + TracingField.REPO, + TracingField.PR_NUMBER, + TracingField.CI_STATUS, + TracingField.EVENT_SOURCE, + TracingField.EVENT_TYPE, + TracingField.LLM_MODEL, + ], + ) + def test_tag_eligible_fields(self, field: TracingField) -> None: + assert field.tag_eligible is True + + @pytest.mark.parametrize( + "field", + [ + TracingField.RETRY_COUNT, + TracingField.SYSTEM_PROMPT_LENGTH, + ], + ) + def test_metadata_only_fields(self, field: TracingField) -> None: + assert field.tag_eligible is False + + +class TestParseTraceFields: + """Config string parsing and validation.""" + + def test_valid_metadata_fields(self) -> None: + fields = parse_trace_fields("ticket_key,ticket_type,retry_count", allow_tags=False) + assert fields == [ + TracingField.TICKET_KEY, + TracingField.TICKET_TYPE, + TracingField.RETRY_COUNT, + ] + + def test_valid_tag_fields(self) -> None: + fields = parse_trace_fields("ticket_type,project_id", allow_tags=True) + assert fields == [TracingField.TICKET_TYPE, TracingField.PROJECT_ID] + + def test_empty_string_returns_empty_list(self) -> None: + assert parse_trace_fields("", allow_tags=True) == [] + + def test_whitespace_only_returns_empty_list(self) -> None: + assert parse_trace_fields(" , , ", allow_tags=True) == [] + + def test_strips_whitespace(self) -> None: + fields = parse_trace_fields(" ticket_key , ticket_type ", allow_tags=False) + assert fields == [TracingField.TICKET_KEY, TracingField.TICKET_TYPE] + + def test_invalid_name_warns_and_skips(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING): + fields = parse_trace_fields("ticket_key,foobar,ticket_type", allow_tags=False) + assert fields == [TracingField.TICKET_KEY, TracingField.TICKET_TYPE] + assert "foobar" in caplog.text + assert "not a recognized field name" in caplog.text + + def test_tag_ineligible_field_in_tags_warns_and_skips( + self, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + fields = parse_trace_fields("ticket_type,retry_count,project_id", allow_tags=True) + assert fields == [TracingField.TICKET_TYPE, TracingField.PROJECT_ID] + assert "retry_count" in caplog.text + assert "not eligible for tags" in caplog.text + + def test_tag_ineligible_field_allowed_in_metadata(self) -> None: + fields = parse_trace_fields("retry_count,system_prompt_length", allow_tags=False) + assert fields == [TracingField.RETRY_COUNT, TracingField.SYSTEM_PROMPT_LENGTH] + + def test_all_invalid_returns_empty(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING): + fields = parse_trace_fields("bad1,bad2", allow_tags=True) + assert fields == [] + + def test_duplicate_fields_preserved(self) -> None: + fields = parse_trace_fields("ticket_key,ticket_key", allow_tags=False) + assert fields == [TracingField.TICKET_KEY, TracingField.TICKET_KEY] + + +def _make_settings(**overrides: Any) -> Settings: + """Build a Settings instance with required fields and overrides.""" + defaults: dict[str, Any] = { + "jira_base_url": "https://test.atlassian.net", + "jira_api_token": "test", + "jira_user_email": "test@example.com", + "github_token": "test", + "anthropic_api_key": "test", + "langfuse_trace_tags": "", + "langfuse_trace_metadata": "", + } + defaults.update(overrides) + return Settings(**defaults) + + +class TestSettingsTraceFields: + """Settings properties parse and validate trace field config.""" + + def test_trace_tag_fields_default_empty(self) -> None: + settings = _make_settings() + assert settings.trace_tag_fields == [] + + def test_trace_metadata_fields_default_empty(self) -> None: + settings = _make_settings() + assert settings.trace_metadata_fields == [] + + def test_trace_tag_fields_parsed(self) -> None: + settings = _make_settings(langfuse_trace_tags="ticket_type,project_id") + assert settings.trace_tag_fields == [TracingField.TICKET_TYPE, TracingField.PROJECT_ID] + + def test_trace_metadata_fields_parsed(self) -> None: + settings = _make_settings(langfuse_trace_metadata="ticket_key,retry_count") + assert settings.trace_metadata_fields == [ + TracingField.TICKET_KEY, + TracingField.RETRY_COUNT, + ] + + def test_tag_ineligible_field_rejected_from_tags( + self, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + settings = _make_settings(langfuse_trace_tags="retry_count,ticket_type") + assert settings.trace_tag_fields == [TracingField.TICKET_TYPE] + + def test_info_logged_when_fields_configured(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.INFO): + settings = _make_settings(langfuse_trace_tags="ticket_type") + _ = settings.trace_tag_fields + assert "Langfuse trace tags configured: ticket_type" in caplog.text + + def test_no_info_logged_when_empty(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.INFO): + settings = _make_settings(langfuse_trace_tags="") + _ = settings.trace_tag_fields + assert "Langfuse trace tags configured" not in caplog.text + + def test_no_info_logged_when_all_invalid(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING): + settings = _make_settings(langfuse_trace_tags="bad1,bad2") + _ = settings.trace_tag_fields + assert "Langfuse trace tags configured" not in caplog.text + + +class TestResolveTraceFields: + """Integration: resolve configured fields from workflow state.""" + + def test_resolves_tags_and_metadata(self) -> None: + state = _make_state() + tag_fields = [TracingField.TICKET_TYPE, TracingField.PROJECT_ID, TracingField.WORKFLOW_STEP] + metadata_fields = [TracingField.TICKET_KEY, TracingField.RETRY_COUNT] + + with ( + patch( + "forge.config.get_settings" + ) as mock_get_settings, + ): + mock_settings = mock_get_settings.return_value + type(mock_settings).trace_tag_fields = PropertyMock(return_value=tag_fields) + type(mock_settings).trace_metadata_fields = PropertyMock(return_value=metadata_fields) + + tags, metadata = resolve_trace_fields(state) + + assert tags == ["Bug", "PROJ", "analyze_bug"] + assert metadata == {"ticket_key": "PROJ-42", "retry_count": "3"} + + def test_skips_missing_fields(self) -> None: + state = _make_state() + del state["current_repo"] + tag_fields = [TracingField.TICKET_TYPE, TracingField.REPO] + metadata_fields = [TracingField.PR_NUMBER] + + with patch( + "forge.config.get_settings" + ) as mock_get_settings: + mock_settings = mock_get_settings.return_value + type(mock_settings).trace_tag_fields = PropertyMock(return_value=tag_fields) + type(mock_settings).trace_metadata_fields = PropertyMock(return_value=metadata_fields) + + tags, metadata = resolve_trace_fields(state) + + assert tags == ["Bug"] + assert metadata == {"pr_number": "99"} + + def test_empty_config_returns_empty(self) -> None: + with patch( + "forge.config.get_settings" + ) as mock_get_settings: + mock_settings = mock_get_settings.return_value + type(mock_settings).trace_tag_fields = PropertyMock(return_value=[]) + type(mock_settings).trace_metadata_fields = PropertyMock(return_value=[]) + + tags, metadata = resolve_trace_fields(_make_state()) + + assert tags == [] + assert metadata == {} + + def test_system_prompt_length_in_metadata(self) -> None: + state = _make_state(system_prompt_length=4523) + metadata_fields = [TracingField.SYSTEM_PROMPT_LENGTH] + + with patch( + "forge.config.get_settings" + ) as mock_get_settings: + mock_settings = mock_get_settings.return_value + type(mock_settings).trace_tag_fields = PropertyMock(return_value=[]) + type(mock_settings).trace_metadata_fields = PropertyMock(return_value=metadata_fields) + + tags, metadata = resolve_trace_fields(state) + + assert tags == [] + assert metadata == {"system_prompt_length": "4523"} + + def test_llm_model_in_tags(self) -> None: + state = _make_state(llm_model="claude-sonnet-4-6-20250514") + tag_fields = [TracingField.LLM_MODEL] + + with patch( + "forge.config.get_settings" + ) as mock_get_settings: + mock_settings = mock_get_settings.return_value + type(mock_settings).trace_tag_fields = PropertyMock(return_value=tag_fields) + type(mock_settings).trace_metadata_fields = PropertyMock(return_value=[]) + + tags, metadata = resolve_trace_fields(state) + + assert tags == ["claude-sonnet-4-6-20250514"] + assert metadata == {} diff --git a/tests/unit/integrations/langfuse/test_tracing.py b/tests/unit/integrations/langfuse/test_tracing.py new file mode 100644 index 00000000..7f097d7c --- /dev/null +++ b/tests/unit/integrations/langfuse/test_tracing.py @@ -0,0 +1,103 @@ +"""Tests for Langfuse tracing configuration. + +Covers get_langfuse_config() metadata passthrough and +get_langfuse_context() metadata parameter. +""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from forge.integrations.langfuse.tracing import ( + AsyncLangfuseContext, + get_langfuse_config, + get_langfuse_context, +) + + +class TestGetLangfuseConfigMetadata: + """get_langfuse_config() includes metadata in _langfuse_context.""" + + def _call_with_handler(self, **kwargs: Any) -> dict[str, Any]: + """Call get_langfuse_config with a mocked handler so it returns a config.""" + with patch( + "forge.integrations.langfuse.tracing.get_langfuse_handler", + return_value=MagicMock(), + ): + return get_langfuse_config(**kwargs) + + def test_metadata_included_in_langfuse_context(self) -> None: + metadata = {"ticket_key": "PROJ-42", "retry_count": "3"} + config = self._call_with_handler(metadata=metadata) + + assert config["_langfuse_context"]["metadata"] == metadata + + def test_metadata_none_when_not_provided(self) -> None: + config = self._call_with_handler(trace_name="test-trace") + assert config["_langfuse_context"]["metadata"] is None + + def test_metadata_merged_into_top_level_metadata(self) -> None: + metadata = {"system_prompt_length": "4523"} + config = self._call_with_handler( + trace_name="test-trace", + metadata=metadata, + ) + assert config["metadata"]["langfuse_trace_name"] == "test-trace" + assert config["metadata"]["system_prompt_length"] == "4523" + + def test_tags_and_metadata_both_passed_through(self) -> None: + tags = ["PROJ-42", "Bug"] + metadata = {"retry_count": "1"} + config = self._call_with_handler( + tags=tags, + metadata=metadata, + session_id="PROJ-42", + ) + ctx = config["_langfuse_context"] + assert ctx["tags"] == ["PROJ-42", "Bug"] + assert ctx["metadata"] == {"retry_count": "1"} + assert ctx["session_id"] == "PROJ-42" + + def test_returns_empty_dict_when_disabled(self) -> None: + with patch( + "forge.integrations.langfuse.tracing.get_langfuse_handler", + return_value=None, + ): + config = get_langfuse_config( + metadata={"key": "val"}, + tags=["tag"], + ) + assert config == {} + + def test_all_context_params_present(self) -> None: + config = self._call_with_handler( + session_id="sess-1", + user_id="user-1", + tags=["t1"], + metadata={"k": "v"}, + ) + ctx = config["_langfuse_context"] + assert ctx["session_id"] == "sess-1" + assert ctx["user_id"] == "user-1" + assert ctx["tags"] == ["t1"] + assert ctx["metadata"] == {"k": "v"} + + +class TestGetLangfuseContext: + """get_langfuse_context() accepts metadata parameter.""" + + def test_creates_context_with_metadata(self) -> None: + ctx = get_langfuse_context( + session_id="sess-1", + tags=["tag"], + metadata={"key": "val"}, + ) + assert isinstance(ctx, AsyncLangfuseContext) + assert ctx.metadata == {"key": "val"} + assert ctx.tags == ["tag"] + assert ctx.session_id == "sess-1" + + def test_metadata_defaults_to_none(self) -> None: + ctx = get_langfuse_context() + assert ctx.metadata is None diff --git a/tests/unit/workflow/nodes/test_pr_creation_trace_context.py b/tests/unit/workflow/nodes/test_pr_creation_trace_context.py new file mode 100644 index 00000000..d3257b40 --- /dev/null +++ b/tests/unit/workflow/nodes/test_pr_creation_trace_context.py @@ -0,0 +1,65 @@ +"""Tests for trace context separation in PR creation helpers.""" + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from forge.workflow.feature.state import create_initial_feature_state +from forge.workflow.nodes.pr_creation import _generate_pr_body_with_agent + + +@pytest.mark.asyncio +async def test_generate_pr_body_splits_prompt_context_from_trace_context() -> None: + mock_jira = MagicMock() + mock_jira.get_issue = AsyncMock() + mock_issue = MagicMock() + mock_issue.summary = "Task summary" + mock_issue.description = "Task description" + mock_jira.get_issue.return_value = mock_issue + + mock_git = MagicMock() + mock_git._run_git.return_value = MagicMock(stdout="abc123 Test commit") + + mock_agent = MagicMock() + mock_agent.run_task = AsyncMock(return_value="# PR Body\n\n" + "x" * 120) + mock_agent._strip_preamble.side_effect = lambda text: text + + state = create_initial_feature_state( + ticket_key="FEAT-123", + ticket_type="Feature", + current_node="create_pr", + current_repo="owner/repo", + current_pr_number=456, + ci_status="pending", + retry_count=2, + ) + state["workspace_path"] = str(Path("/tmp/test-workspace")) + state["context"] = {"source": "jira"} + + with patch("forge.workflow.nodes.pr_creation.ForgeAgent", return_value=mock_agent): + result = await _generate_pr_body_with_agent( + state, + mock_git, + mock_jira, + ["TASK-1"], + ) + + assert result is not None + call_kwargs = mock_agent.run_task.call_args.kwargs + + assert call_kwargs["context"] == { + "ticket_key": "FEAT-123", + "task_count": 1, + } + assert call_kwargs["trace_context"] == { + "ticket_key": "FEAT-123", + "ticket_type": "Feature", + "current_node": "create_pr", + "repo": "owner/repo", + "pr_number": 456, + "ci_status": "pending", + "event_type": "", + "event_source": "jira", + "retry_count": 2, + } diff --git a/tests/unit/workflow/nodes/test_trace_context_enrichment.py b/tests/unit/workflow/nodes/test_trace_context_enrichment.py new file mode 100644 index 00000000..eef9af36 --- /dev/null +++ b/tests/unit/workflow/nodes/test_trace_context_enrichment.py @@ -0,0 +1,435 @@ +"""Tests that workflow nodes pass trace-related context fields to the agent. + +Each workflow node was updated to include trace fields (ticket_type, current_node, +event_type, event_source, retry_count, etc.) in the context dict passed to the +agent. These tests verify the context dicts contain the expected trace keys. +""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from forge.models.workflow import TicketType + +TRACE_CONTEXT_KEYS = { + "ticket_key", + "ticket_type", + "current_node", + "event_type", + "event_source", + "retry_count", +} + + +def _make_feature_state(**overrides: Any) -> dict[str, Any]: + """Build a minimal feature workflow state.""" + from forge.workflow.feature.state import create_initial_feature_state + + state = create_initial_feature_state( + ticket_key="TEST-123", + ticket_type=TicketType.FEATURE, + ) + state["current_node"] = "generate_prd" + state["event_type"] = "issue_updated" + state["context"] = {"source": "jira"} + state["retry_count"] = 0 + state.update(overrides) + return state + + +def _make_bug_state(**overrides: Any) -> dict[str, Any]: + """Build a minimal bug workflow state.""" + state: dict[str, Any] = { + "thread_id": "test-thread", + "ticket_key": "BUG-456", + "ticket_type": "Bug", + "current_node": "analyze_bug", + "event_type": "issue_updated", + "context": {"source": "jira"}, + "ci_status": "", + "retry_count": 0, + "is_paused": False, + "error_message": None, + } + state.update(overrides) + return state + + +class TestPrdGenerationTraceContext: + """generate_prd node includes trace fields in agent context.""" + + @pytest.mark.asyncio + async def test_generate_prd_passes_trace_fields(self) -> None: + from forge.workflow.nodes.prd_generation import generate_prd + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.get_issue = AsyncMock( + return_value=MagicMock( + summary="Test Feature", + description="Build something", + project_key="TEST", + ) + ) + mock_jira.add_comment = AsyncMock() + mock_jira.add_structured_comment = AsyncMock() + mock_jira.update_description = AsyncMock() + mock_jira.set_workflow_label = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_generate_prd(raw_req, context=None): + if context: + captured_context.update(context) + return "# PRD\n\nContent" + + mock_agent.generate_prd = capture_generate_prd + + state = _make_feature_state(current_node="generate_prd") + + with ( + patch( + "forge.workflow.nodes.prd_generation.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.prd_generation.ForgeAgent", + return_value=mock_agent, + ), + ): + await generate_prd(state) + + for key in TRACE_CONTEXT_KEYS: + assert key in captured_context, f"Missing trace key '{key}' in PRD context" + + @pytest.mark.asyncio + async def test_regenerate_prd_passes_trace_fields(self) -> None: + from forge.workflow.nodes.prd_generation import regenerate_prd_with_feedback + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.update_description = AsyncMock() + mock_jira.set_workflow_label = AsyncMock() + mock_jira.add_comment = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_regen(**kwargs): + if kwargs.get("context"): + captured_context.update(kwargs["context"]) + return "# Revised PRD" + + mock_agent.regenerate_with_feedback = capture_regen + + state = _make_feature_state( + current_node="regenerate_prd", + prd_content="# Old PRD", + feedback_comment="Add more detail", + revision_requested=True, + ) + + with ( + patch( + "forge.workflow.nodes.prd_generation.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.prd_generation.ForgeAgent", + return_value=mock_agent, + ), + ): + await regenerate_prd_with_feedback(state) + + for key in {"ticket_type", "current_node", "event_type", "event_source", "retry_count"}: + assert key in captured_context, f"Missing trace key '{key}' in PRD regen context" + + +class TestSpecGenerationTraceContext: + """generate_spec node includes trace fields in agent context.""" + + @pytest.mark.asyncio + async def test_generate_spec_passes_trace_fields(self) -> None: + from forge.workflow.nodes.spec_generation import generate_spec + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.update_description = AsyncMock() + mock_jira.set_workflow_label = AsyncMock() + mock_jira.add_comment = AsyncMock() + mock_jira.add_structured_comment = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_generate_spec(prd, context=None): + if context: + captured_context.update(context) + return "# Spec\n\nContent" + + mock_agent.generate_spec = capture_generate_spec + + state = _make_feature_state( + current_node="generate_spec", + prd_content="# PRD content", + qa_history=[], + ) + + with ( + patch( + "forge.workflow.nodes.spec_generation.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.spec_generation.ForgeAgent", + return_value=mock_agent, + ), + patch("forge.workflow.nodes.spec_generation.post_qa_summary_if_needed"), + ): + await generate_spec(state) + + for key in TRACE_CONTEXT_KEYS: + assert key in captured_context, f"Missing trace key '{key}' in spec context" + + @pytest.mark.asyncio + async def test_regenerate_spec_passes_trace_fields(self) -> None: + from forge.workflow.nodes.spec_generation import regenerate_spec_with_feedback + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.update_description = AsyncMock() + mock_jira.set_workflow_label = AsyncMock() + mock_jira.add_comment = AsyncMock() + mock_jira.add_structured_comment = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_regen(**kwargs): + if kwargs.get("context"): + captured_context.update(kwargs["context"]) + return "# Revised Spec" + + mock_agent.regenerate_with_feedback = capture_regen + + state = _make_feature_state( + current_node="regenerate_spec", + spec_content="# Old Spec", + prd_content="# PRD", + feedback_comment="Change approach", + revision_requested=True, + ) + + with ( + patch( + "forge.workflow.nodes.spec_generation.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.spec_generation.ForgeAgent", + return_value=mock_agent, + ), + ): + await regenerate_spec_with_feedback(state) + + for key in {"ticket_type", "current_node", "event_type", "event_source", "retry_count"}: + assert key in captured_context, f"Missing trace key '{key}' in spec regen context" + + +class TestQaHandlerTraceContext: + """answer_question node includes trace fields in agent context.""" + + @pytest.mark.asyncio + async def test_answer_question_passes_trace_fields(self) -> None: + from forge.workflow.nodes.qa_handler import answer_question + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.add_comment = AsyncMock(return_value=MagicMock(id="c-1")) + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_answer(question, artifact_content, context): + captured_context.update(context) + return "The answer" + + mock_agent.answer_question = capture_answer + + state = _make_feature_state( + current_node="prd_approval_gate", + feedback_comment="?Why this approach?", + is_question=True, + prd_content="# PRD", + ) + + with ( + patch( + "forge.workflow.nodes.qa_handler.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.qa_handler.ForgeAgent", + return_value=mock_agent, + ), + ): + await answer_question(state) + + for key in TRACE_CONTEXT_KEYS: + assert key in captured_context, f"Missing trace key '{key}' in QA context" + + +class TestEpicDecompositionTraceContext: + """decompose_epics node includes trace fields in agent context.""" + + @pytest.mark.asyncio + async def test_decompose_epics_passes_trace_fields(self) -> None: + from forge.workflow.nodes.epic_decomposition import decompose_epics + + mock_jira = AsyncMock() + mock_jira.get_issue = AsyncMock( + return_value=MagicMock( + project_key="TEST", + summary="Test Feature", + ) + ) + mock_jira.get_labels = AsyncMock(return_value=[]) + mock_jira.get_project_repos = AsyncMock(return_value=["acme/backend"]) + mock_jira.create_epic = AsyncMock(return_value="TEST-200") + mock_jira.set_workflow_label = AsyncMock() + mock_jira.add_comment = AsyncMock() + + mock_agent = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_epics(spec, context=None): + if context: + captured_context.update(context) + return [{"summary": "Epic 1", "plan": "Do it", "repo": "acme/backend"}] + + mock_agent.generate_epics = capture_epics + + state = _make_feature_state( + current_node="decompose_epics", + spec_content="# Spec content", + generation_context={}, + qa_history=[], + ) + + with ( + patch( + "forge.workflow.nodes.epic_decomposition.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.epic_decomposition.ForgeAgent", + return_value=mock_agent, + ), + patch("forge.workflow.nodes.epic_decomposition.post_qa_summary_if_needed"), + ): + await decompose_epics(state) + + for key in TRACE_CONTEXT_KEYS: + assert key in captured_context, f"Missing trace key '{key}' in epic context" + + @pytest.mark.asyncio + async def test_update_single_epic_passes_trace_fields(self) -> None: + from forge.workflow.nodes.epic_decomposition import update_single_epic + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.get_issue = AsyncMock( + return_value=MagicMock(description="Original epic") + ) + mock_jira.update_description = AsyncMock() + mock_jira.add_comment = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_regen(**kwargs): + if kwargs.get("context"): + captured_context.update(kwargs["context"]) + return "# Revised Epic" + + mock_agent.regenerate_with_feedback = capture_regen + + state = _make_feature_state( + current_node="update_single_epic", + epic_keys=["TEST-200"], + current_epic_key="TEST-200", + feedback_comment="Change scope", + revision_requested=True, + ) + + with ( + patch( + "forge.workflow.nodes.epic_decomposition.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.epic_decomposition.ForgeAgent", + return_value=mock_agent, + ), + ): + await update_single_epic(state) + + for key in {"ticket_type", "current_node", "event_type", "event_source", "retry_count"}: + assert key in captured_context, f"Missing trace key '{key}' in epic update context" + + +class TestTaskGenerationTraceContext: + """update_single_task node includes trace fields in agent context.""" + + @pytest.mark.asyncio + async def test_update_single_task_passes_trace_fields(self) -> None: + from forge.workflow.nodes.task_generation import update_single_task + + mock_jira = MagicMock() + mock_jira.close = AsyncMock() + mock_jira.get_issue = AsyncMock( + return_value=MagicMock(description="Original task") + ) + mock_jira.update_description = AsyncMock() + mock_jira.add_comment = AsyncMock() + + mock_agent = MagicMock() + mock_agent.close = AsyncMock() + captured_context: dict[str, Any] = {} + + async def capture_regen(**kwargs): + if kwargs.get("context"): + captured_context.update(kwargs["context"]) + return "# Revised Task" + + mock_agent.regenerate_with_feedback = capture_regen + + state = _make_feature_state( + current_node="update_single_task", + current_task_key="TEST-300", + feedback_comment="Make it smaller", + revision_requested=True, + ) + + with ( + patch( + "forge.workflow.nodes.task_generation.JiraClient", + return_value=mock_jira, + ), + patch( + "forge.workflow.nodes.task_generation.ForgeAgent", + return_value=mock_agent, + ), + ): + await update_single_task(state) + + for key in {"ticket_type", "current_node", "event_type", "event_source", "retry_count"}: + assert key in captured_context, f"Missing trace key '{key}' in task update context"