initial version of cost metrics in cli

README.md

@@ -217,6 +217,49 @@ Results are automatically saved to a structured directory in the following forma
 | `--compose-file`| | Custom docker-compose.yml; overrides the built-in default |
 | `--keep-server` | `false` | Leave the vLLM server running after the benchmark |
 | `--server-timeout`| `900` | Seconds to wait for the vLLM server to become ready |
+| `--cost` | `true` | Compute rent-vs-buy cost metrics and embed them in the result JSON |
+| `--system-bundle` | | Named system bundle to price the owned node as an all-in system (e.g. `PRU2500_8x5090`) |
+| `--pue` | *(config default)* | Power Usage Effectiveness for owned-side electricity |
+| `--avg-gpu-watts` | *(sampled)* | Override GPU power draw (W/GPU); `0` samples live via `nvidia-smi` |
+| `--pricing-file` | *(embedded)* | External `hardware_pricing.json` to override the embedded pricing config |
+
+---
+
+### Cost Metrics (Rent vs Buy)
+
+When `--cost` is enabled (default), each benchmark run is enriched with a `cost_metrics`
+block in `benchmark_result.json` and surfaced in the dashboard. It answers "is it cheaper
+to own the CoreSpan node or rent equivalent GPUs?" using the run's real duration, token
+counts, GPU count (TP×PP), and GPU power.
+
+**Owned cost is fully loaded** — amortized hardware CapEx **plus electricity** (PUE-adjusted,
+US industrial rate) — so it is a fair comparison against rental rates, which already bundle power.
+
+- **Power** is measured live during the run via `nvidia-smi`, scoped to the **GPUs in use**
+  (the busiest `TP×PP` GPUs by draw) so electricity stays consistent with the per-GPU CapEx.
+  If sampling is unavailable it falls back to the per-GPU default in the pricing config, or to
+  `--avg-gpu-watts`.
+- **System bundles** contribute a **per-GPU** all-in price scaled by the GPUs the run actually
+  used, so a 4-GPU run (`TP1×PP4`, the default compose) against the 8-GPU `PRU2500_8x5090` bundle
+  prices 4 GPUs — while a full 8-GPU run reproduces the whole-node bundle price exactly.
+- Output includes cost-per-1M-tokens (3:1 output/input weighting) and a rent-vs-buy ladder across
+  market tiers (spot → premium), each with a `buy`/`rent` verdict.
+
+**Supported GPUs** (auto-detected from `nvidia-smi`): RTX 5090, RTX 5070, Tesla T4, A100, H100 —
+each carries a full rent-vs-buy tier ladder (others fall back to `DEFAULT` pricing). The
+RTX 5090 + `PRU2500_8x5090` bundle reproduces the published
+[rent-or-buy blog](https://www.corespan.ai/resources/blog/rent-or-buy-rtx-5090-pru-2500) figures.
+
+```bash
+# Price the owned side as the all-in PRU-2500 + 8x RTX 5090 system
+ai-studio-cli bench --requests 500 --concurrency 32 --system-bundle PRU2500_8x5090
+
+# Use a fixed power figure and a custom pricing file instead of live sampling
+ai-studio-cli bench --avg-gpu-watts 550 --pricing-file ./hardware_pricing.json
+```
+
+> Cost dollars scale with the run window, so a short benchmark shows small amounts; the
+> cost-per-1M-tokens and the buy/rent verdict are the run-scoped signals to read.
 
 ---
 
@@ -234,8 +277,9 @@ ai-studio-cli bench-ui --result-dir /path/to/bench-results
 ```
 
 The dashboard provides:
-- **KPI cards** — Total throughput, output throughput, TTFT, and TPOT at a glance
+- **KPI cards** — Total throughput, output throughput, TTFT, TPOT, and the rent-vs-buy verdict (at the market-median tier) at a glance
 - **Performance trend chart** — Visualise metrics across runs with configurable Y-axis
+- **Cost analysis panel** — Owned (fully-loaded) cost, electricity, blended $/1M tokens, avg GPU power, and the full rent-vs-buy tier ladder with per-tier `buy`/`rent` verdicts
 - **Model config bar** — Shows TP/PP, quantization, dtype, max model length, GPU memory utilization
 - **Filterable table** — Filter by model, GPU, and precision; search across all runs
 - **Sidebar navigation** — Quickly switch between benchmark runs

ai-studio-cli/cmd/bench.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/corespan/ai-studio-cli/internal/power"
 	"github.com/spf13/cobra"
 	"golang.org/x/term"
 )
@@ -52,6 +53,17 @@ type benchRunner struct {
 	gpuTag            string
 	openUI            bool
 	uiPort            int
+
+	// cost metrics
+	costEnabled         bool
+	systemBundle        string
+	pue                 float64
+	pricingFile         string
+	avgGPUWattsOverride float64
+
+	// populated at runtime
+	gpuWattReadings []float64 // per-GPU draw sampled during the run
+	resolvedGPU     string
 }
 
 var bench = &benchRunner{}
@@ -103,6 +115,12 @@ func (r *benchRunner) registerFlags(cmd *cobra.Command) {
 	// Dashboard
 	cmd.Flags().BoolVar(&r.openUI, "ui", true, "On an interactive terminal, launch the results dashboard when the run finishes and print a clickable link (blocks until Ctrl+C); auto-skipped for non-interactive/CI runs, or pass --ui=false")
 	cmd.Flags().IntVar(&r.uiPort, "ui-port", 9090, "Port for the dashboard launched by --ui")
+	// Cost metrics
+	cmd.Flags().BoolVar(&r.costEnabled, "cost", true, "Compute rent-vs-buy cost metrics and embed them in the result JSON")
+	cmd.Flags().StringVar(&r.systemBundle, "system-bundle", "", "Named system bundle from the pricing config (e.g. PRU2500_8x5090) to price the owned node as an all-in system")
+	cmd.Flags().Float64Var(&r.pue, "pue", 0, "Power Usage Effectiveness for owned-side electricity (0 = use pricing config default)")
+	cmd.Flags().StringVar(&r.pricingFile, "pricing-file", "", "Path to an external hardware_pricing.json to override the embedded pricing config")
+	cmd.Flags().Float64Var(&r.avgGPUWattsOverride, "avg-gpu-watts", 0, "Override GPU power draw (watts per GPU) for cost energy; 0 = sample live via nvidia-smi")
 }
 
 // entry point
@@ -155,6 +173,7 @@ func (r *benchRunner) runBenchmark(cmd *cobra.Command, composePath string) error
 	if gpu == "" {
 		gpu = detectGPUTag()
 	}
+	r.resolvedGPU = gpu
 
 	timestamp := time.Now().Format("2006-01-02T15-04-05")
 	structuredDir := filepath.Join(r.resultDir, sanitizePath(modelName), sanitizePath(gpu), timestamp)
@@ -174,8 +193,19 @@ func (r *benchRunner) runBenchmark(cmd *cobra.Command, composePath string) error
 		return fmt.Errorf("creating result directory: %w", err)
 	}
 
-	if err := r.execute(cmd.Context(), args, structuredDir); err != nil {
-		return err
+	// Sample GPU power during the run so the cost engine can charge realistic
+	// owned-side electricity (no telemetry stack exists otherwise).
+	var sampler *power.Sampler
+	if r.costEnabled && r.avgGPUWattsOverride <= 0 {
+		sampler = power.NewSampler(2 * time.Second)
+		sampler.Start()
+	}
+	runErr := r.execute(cmd.Context(), args, structuredDir)
+	if sampler != nil {
+		r.gpuWattReadings = sampler.Stop()
+	}
+	if runErr != nil {
+		return runErr
 	}
 
 	resultPath := filepath.Join(structuredDir, r.resultFilename)

ai-studio-cli/cmd/cost.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/corespan/ai-studio-cli/internal/cost"
+)
+
+// injectCostMetrics computes rent-vs-buy cost metrics from the benchmark result
+// and adds them under raw["cost_metrics"]. It reads token/duration figures from
+// the vLLM result, GPU count from the model config (TP*PP), and owned-side power
+// from the live sample (falling back to the pricing config's default wattage).
+func (r *benchRunner) injectCostMetrics(raw map[string]interface{}, cfg ModelConfig) error {
+	engine, err := cost.NewEngine(r.pricingFile)
+	if err != nil {
+		return fmt.Errorf("loading pricing config: %w", err)
+	}
+
+	duration := toFloat(raw["duration"])
+	inTok := toFloat(raw["total_input_tokens"])
+	outTok := toFloat(raw["total_output_tokens"])
+	if duration <= 0 || (inTok+outTok) <= 0 {
+		return fmt.Errorf("result missing duration/token fields; skipping cost metrics")
+	}
+
+	numGPUs := cfg.TensorParallel * cfg.PipelineParallel
+	if numGPUs < 1 {
+		numGPUs = 1
+	}
+
+	gpuName := r.resolvedGPU
+	if gpuName == "" {
+		gpuName = detectGPUTag()
+	}
+
+	// Per-GPU power precedence: explicit override > live sample of the GPUs in
+	// use > per-GPU default from the pricing config.
+	perGPUWatts := r.avgGPUWattsOverride
+	if perGPUWatts <= 0 {
+		perGPUWatts = avgWattsOfUsedGPUs(r.gpuWattReadings, numGPUs)
+	}
+	if perGPUWatts <= 0 {
+		perGPUWatts = engine.DefaultWatts(gpuName)
+	}
+
+	result := engine.CalculateLLM(cost.LLMInput{
+		GPUName:           gpuName,
+		NumGPUs:           numGPUs,
+		DurationSeconds:   duration,
+		AvgPowerWatts:     perGPUWatts,
+		TotalInputTokens:  inTok,
+		TotalOutputTokens: outTok,
+		Infra: cost.InfraConfig{
+			SystemBundle: r.systemBundle,
+			PUE:          r.pue,
+		},
+	})
+
+	raw["cost_metrics"] = result
+	return nil
+}
+
+// avgWattsOfUsedGPUs returns the mean per-GPU draw of the busiest `used` GPUs.
+// The GPUs doing the work draw the most power, so ranking by draw and taking the
+// top N approximates "the GPUs in use" without needing device-ID mapping — and
+// keeps electricity scoped to the same GPUs the CapEx is scoped to.
+func avgWattsOfUsedGPUs(readings []float64, used int) float64 {
+	if len(readings) == 0 || used <= 0 {
+		return 0
+	}
+	sorted := append([]float64(nil), readings...)
+	sort.Sort(sort.Reverse(sort.Float64Slice(sorted)))
+	if used > len(sorted) {
+		used = len(sorted)
+	}
+	var sum float64
+	for i := 0; i < used; i++ {
+		sum += sorted[i]
+	}
+	return sum / float64(used)
+}

ai-studio-cli/cmd/metadata.go

@@ -32,6 +32,14 @@ func (r *benchRunner) patchResultJSON(resultPath, composePath string) error {
 		raw["total_token_throughput"] = outTP + inTP
 	}
 
+	if r.costEnabled {
+		if err := r.injectCostMetrics(raw, cfg); err != nil {
+			fmt.Printf("  [cost] warning: %v\n", err)
+		} else {
+			fmt.Println("  [cost] rent-vs-buy metrics added to result JSON")
+		}
+	}
+
 	out, err := json.MarshalIndent(raw, "", "  ")
 	if err != nil {
 		return fmt.Errorf("marshalling patched result: %w", err)

ai-studio-cli/internal/benchui/ui/app.js

@@ -37,6 +37,7 @@
     valOutputThroughput: $('#val-output-throughput'),
     valTtft: $('#val-ttft'),
     valTpot: $('#val-tpot'),
+    valVerdict: $('#val-verdict'),
   };
 
   // ----- API -----
@@ -233,6 +234,7 @@
       state.selectedRun = await fetchRunDetail(model, gpu, timestamp, signal);
       renderKPIs();
       renderConfigBar();
+      renderCost();
     } catch (err) {
       if (err.name === 'AbortError') return;
       console.error('Error loading run detail:', err);
@@ -248,6 +250,24 @@
     dom.valOutputThroughput.textContent = fmtNum(r.output_throughput);
     dom.valTtft.textContent = fmtNum(r.mean_ttft_ms);
     dom.valTpot.textContent = fmtNum(r.mean_tpot_ms);
+    renderVerdictKPI();
+  }
+
+  // Rent-vs-buy verdict at the market-median tier, shown in the top KPI row.
+  function renderVerdictKPI() {
+    const r = state.selectedRun;
+    const cm = r && r.cost_metrics;
+    const ladder = cm && cm.rent_vs_buy_analysis;
+    const tiers = ladder && Array.isArray(ladder.tiers) ? ladder.tiers : [];
+    const median = tiers.find(t => t.tier === 'dedicated_median');
+    if (!median) {
+      dom.valVerdict.textContent = '—';
+      dom.valVerdict.style.color = '';
+      return;
+    }
+    const buy = median.verdict === 'buy';
+    dom.valVerdict.textContent = buy ? 'BUY' : 'RENT';
+    dom.valVerdict.style.color = buy ? 'hsl(142, 60%, 55%)' : 'hsl(38, 90%, 60%)';
   }
 
   // ----- Config Bar -----
@@ -278,6 +298,59 @@
     dom.cfgGmem.textContent = mc.gpu_memory_utilization ? `mem ${mc.gpu_memory_utilization}` : '';
   }
 
+  // ----- Cost Analysis -----
+  function renderCost() {
+    const r = state.selectedRun;
+    const cm = r && r.cost_metrics;
+    const sec = $('#cost-section');
+    if (!sec) return;
+
+    const ladder = cm && cm.rent_vs_buy_analysis;
+    if (!cm || !ladder || !Array.isArray(ladder.tiers) || ladder.tiers.length === 0) {
+      sec.classList.add('hidden');
+      return;
+    }
+    sec.classList.remove('hidden');
+
+    $('#cost-owned').textContent = fmtUSD(ladder.owned_fully_loaded_usd);
+    $('#cost-energy').textContent = fmtUSD(ladder.owned_energy_cost_usd);
+    $('#cost-blended').textContent = cm.corespan_infra
+      ? fmtUSD(cm.corespan_infra.blended_cost_per_1m_tokens_usd, 2)
+      : '—';
+    $('#cost-watts').textContent = fmtNum(cm.avg_gpu_power_watts);
+
+    $('#cost-ladder-body').innerHTML = ladder.tiers.map(t => {
+      const buy = t.verdict === 'buy';
+      const cls = buy ? 'verdict-buy' : 'verdict-rent';
+      const sign = Number(t.savings_usd_vs_buy) >= 0 ? '+' : '−';
+      const mag = fmtUSD(Math.abs(Number(t.savings_usd_vs_buy)));
+      return `<tr>
+        <td>${esc(t.label || t.tier)}</td>
+        <td class="mono">$${fmtNum(t.rental_hourly_usd)}</td>
+        <td class="mono">${fmtUSD(t.rental_total_usd)}</td>
+        <td class="mono ${cls}">${sign}${mag}</td>
+        <td><span class="verdict-chip ${cls}">${buy ? 'BUY' : 'RENT'}</span></td>
+      </tr>`;
+    }).join('');
+
+    const median = ladder.tiers.find(t => t.tier === 'dedicated_median');
+    const vb = $('#cost-verdict');
+    if (median) {
+      const buy = median.verdict === 'buy';
+      vb.textContent = buy ? 'BUY wins at market median' : 'RENT wins at market median';
+      vb.className = 'cost-verdict-badge ' + (buy ? 'verdict-buy' : 'verdict-rent');
+    } else {
+      vb.textContent = '';
+      vb.className = 'cost-verdict-badge';
+    }
+
+    const fn = $('#cost-footnote');
+    if (fn) {
+      const capex = fmtUSD(ladder.owned_capex_only_usd);
+      fn.textContent = `Owned = amortized CapEx (${capex}) + electricity over a ${fmtNum(ladder.window_hours)}h window · ${ladder.effective_num_gpus} GPU(s) · pricing as of ${cm.pricing_source_date || 'n/a'}.`;
+    }
+  }
+
   // ----- Chart -----
   function renderChart() {
     const metricKey = dom.filterMetric.value;
@@ -459,6 +532,25 @@
     return isNaN(num) ? '—' : num.toFixed(2);
   }
 
+  // fmtUSD formats a dollar amount. Pass `decimals` to force a fixed precision;
+  // otherwise it adapts — whole dollars for large sums, and enough decimals to
+  // reveal sub-cent values (so short-run costs show the real number, not "$0").
+  function fmtUSD(val, decimals) {
+    if (val == null) return '—';
+    const num = Number(val);
+    if (isNaN(num)) return '—';
+    let d = decimals;
+    if (d == null) {
+      const abs = Math.abs(num);
+      if (abs === 0) d = 2;
+      else if (abs >= 1000) d = 0;              // $111,568
+      else if (abs >= 1) d = 2;                 // $25.09
+      // sub-dollar: ~2 significant figures so small costs show the real value
+      else d = Math.min(8, Math.max(2, 1 - Math.floor(Math.log10(abs))));
+    }
+    return '$' + num.toLocaleString(undefined, { minimumFractionDigits: d, maximumFractionDigits: d });
+  }
+
   function formatTimestamp(ts) {
     if (!ts) return '—';
     return ts.replace(/T/, ' ').replace(/-(\d{2})-(\d{2})$/, ':$1:$2');
@@ -496,6 +588,40 @@
       color: var(--accent);
       border: 1px solid var(--border-accent);
     }
+    #cost-section { margin-top: var(--space-lg, 16px); }
+    .cost-kpi-row {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
+      gap: var(--space-md, 12px);
+      margin-bottom: var(--space-lg, 16px);
+    }
+    .cost-ladder-table td { vertical-align: middle; }
+    .verdict-buy { color: hsl(142, 60%, 55%); }
+    .verdict-rent { color: hsl(38, 90%, 60%); }
+    .verdict-chip {
+      display: inline-flex;
+      align-items: center;
+      padding: 0.15em 0.6em;
+      font-size: 0.72rem;
+      font-weight: 700;
+      font-family: var(--font-mono);
+      border-radius: 999px;
+      border: 1px solid currentColor;
+    }
+    .cost-verdict-badge {
+      font-size: 0.75rem;
+      font-weight: 700;
+      font-family: var(--font-mono);
+      padding: 0.2em 0.7em;
+      border-radius: 999px;
+      border: 1px solid currentColor;
+    }
+    .cost-footnote {
+      margin-top: var(--space-md, 12px);
+      font-size: 0.75rem;
+      color: var(--text-muted);
+      font-family: var(--font-sans);
+    }
     .error-toast {
       position: fixed;
       top: var(--space-lg);