diff --git a/.agents/skills/wandb-tracking/SKILL.md b/.agents/skills/wandb-tracking/SKILL.md new file mode 100644 index 0000000..d55b1f3 --- /dev/null +++ b/.agents/skills/wandb-tracking/SKILL.md @@ -0,0 +1,60 @@ +--- +name: wandb-tracking +description: Read, analyze, and manage Weights & Biases (wandb) experiment data for PithTrain runs. Use when the user pastes a wandb.ai URL, or asks to list runs, compare runs or loss curves (per-step and final delta), check whether a variant matches a baseline, read training throughput (tokens-per-second), pull a run's console log after a crash (output.log, stdout+stderr with tracebacks), or build and manage a saved view of which runs to show. +--- + +# wandb-tracking + +Work with wandb experiment data for PithTrain via the scripts below: runs, metrics, console output, and saved views. + +The scripts below are the primitives; compose them for the question asked. **Do not produce an unsolicited full report.** Answer the specific question. + +## Prerequisites + +- wandb must be authenticated: `wandb.Api()` reads credentials from `~/.netrc`. +- Scripts live in `scripts/` beside this file. + +## Primitives + +| Question | Command | +|---|---| +| What runs / metric keys exist? | `runs.py list ` | +| Per-step + final delta of a metric across runs | `runs.py compare --metric KEY --ref RUN --runs RUN[,RUN...]` | +| Download console output (`output.log`, stdout+stderr) for runs | `runs.py logs --runs RUN[,RUN...]` | +| List views / what each view shows | `views.py list ` | +| What does one `?nw=` view show? | `views.py show ` | +| Create / update a view | `views.py set --show RUN[,RUN...] --name NAME [--update NW]` | +| Delete a view | `views.py delete ` | + +`RUN` is a run **id** or **display name** everywhere. `entity/project` example: `pithtrain/pr74`. + +## Runs and metrics + +Run `runs.py list` first. It prints `id | state | steps | name` and the **union of logged metric keys** to feed `runs.py compare --metric`. PithTrain logs `train/cross-entropy-loss`, `train/gradient-norm`, `train/load-balance-loss`, `train/learning-rate`, `train/step`, `infra/step-time`, `infra/tokens-per-second`, `infra/peak-gpu-memory`. + +### Delta reporting (comparing loss curves) + +`runs.py compare` is the canonical report: it matters for loss curves because a raw final number hides the trajectory. Each `--runs` entry is compared against `--ref` (`delta = run - ref`); for a pairwise A-vs-B use `--ref B --runs A`. It reports, per run: + +- **per-step delta range** across all steps (the extreme is usually an early transient). +- **final delta** at the last step, plus **settling** (mean of the last 5): where it's headed. + +These are facts, not a verdict. Report them to the user as prose plus a small table, and always state the step count / horizon: a 64-step run only rules out *large* effects, so say so, and judge the deltas against that horizon yourself. + +## Throughput + +Throughput is the logged metric `infra/tokens-per-second`. Pass **`--drop-first`**: step 0 is a warmup outlier (first-step compile/alloc). Steady-state fp8-vs-bf16 and pow2-vs-full-mantissa (the two fp8 scale formats) deltas then fall out of `runs.py compare`. (At small scale, expect fp8 *slower* than bf16, since quant overhead outweighs the GEMM matrix-multiply win.) + +## Console output (stdout + stderr) + +`runs.py logs` downloads `output.log` per run: the run's console output (**both stdout and stderr**), captured and synced upstream. A crashed run's traceback lands here too, so this is the first place to look when a run died. `--file` selects a different run file: `config.yaml`, `wandb-metadata.json` (host/git/command/GPU), `wandb-summary.json`, `requirements.txt`. **Offset gotcha:** `output.log` is 1-indexed and starts at step 2; the wandb scalar `_step` is 0-indexed (0..N-1). Cross-reference by content, not by line number. + +## Views (`?nw=`) + +A view (the `?nw=` saved workspace) narrows which of a project's runs are shown. Driving `views.py`: + +- From a pasted `?nw=` URL, pass the `` to `show` (or `list` to see every view's `shown` / `hidden` runs). +- `set --show RUN[,...] --name NAME` creates a view of those runs and prints its `?nw=` URL. `--from-view NW` starts from an existing view's layout/panels; `--update NW` edits a view in place (**same URL**), otherwise a new slug is minted. +- Creating/updating a view is an outward-facing write to the user's project, so confirm the target runs + name first. Reversible with `delete`. + +See [references/views.md](references/views.md) if you need to modify `views.py` itself. diff --git a/.agents/skills/wandb-tracking/references/views.md b/.agents/skills/wandb-tracking/references/views.md new file mode 100644 index 0000000..f0ede38 --- /dev/null +++ b/.agents/skills/wandb-tracking/references/views.md @@ -0,0 +1,61 @@ +# wandb views via GraphQL + +Views are managed through GraphQL: `api._service_api.execute_graphql(query_str, variables)` takes a query string and a variables dict and returns the parsed `data` field. + +## nw slug <-> view name + +- Workspace URL `https://wandb.ai//?nw=`. +- Internal view name is `nw--v`; the human label is `displayName`. +- The default workspace view ends in `-w` (e.g. `nw--w`) rather than `-v`. It is a project-view too, so it appears in the list and works as a clone template. + +## Listing views + +```graphql +query Views($entityName: String!, $projectName: String!) { + project(name: $projectName, entityName: $entityName) { + allViews(viewType: "project-view") { + edges { node { id name displayName spec } } + } + } +} +``` + +`spec` is a JSON string. The runset lives at `spec.section.runSets[0]`. + +## What a view shows (selection semantics) + +- `runSets[0].filters` empty => every run in the project is in the table. +- `runSets[0].selections = {root, bounds, tree}` narrows what is plotted. +- With `root == 1`, `tree` is the list of **hidden** (eye-closed) run ids, so: + + **shown = all_run_ids - tree** + +To narrow a view to a set of runs, set `tree = sorted(all_ids - show_ids)`. + +## Creating / updating (upsertView) + +```graphql +mutation Up($input: UpsertViewInput!) { + upsertView(input: $input) { view { id name displayName } inserted } +} +``` + +`UpsertViewInput` fields that matter: + +- `entityName`, `projectName`, `type: "project-view"` (where it lives). +- `name`: `nw--v`. Mint a fresh 11-char slug for a new view. +- `displayName`: the human label. +- `spec`: JSON string (clone an existing view's spec, then rewrite the tree). +- `createdUsing`: `WANDB_SDK`. +- `id`: pass it to update in place (keeps the same URL); omit to create. + `inserted` in the response is `true` for create, `false` for update. + +Cloning an existing view's spec carries the project's panel layout, so a new view opens on the same charts. + +## Deleting + +```graphql +mutation Del($input: DeleteViewInput!) { deleteView(input: $input) { success } } +``` + +Input is `{ id }`. Views are cheap and reversible, but creating/updating one is a write to the user's project, so confirm the runs and name first. diff --git a/.agents/skills/wandb-tracking/scripts/runs.py b/.agents/skills/wandb-tracking/scripts/runs.py new file mode 100644 index 0000000..232e94d --- /dev/null +++ b/.agents/skills/wandb-tracking/scripts/runs.py @@ -0,0 +1,140 @@ +""" +List, compare, and pull files for runs in a wandb project. + + python scripts/runs.py list + python scripts/runs.py compare --metric KEY --ref RUN --runs RUN[,RUN...] [--drop-first] + python scripts/runs.py logs --runs RUN[,RUN...] [--file NAME] + +RUN is a run id or display name. + +compare reports, per run, the per-step delta range and the final delta with what +it settles into. It reports facts, not a verdict; judge significance yourself. +--drop-first drops step 0 (a warmup outlier for throughput; do not use it for loss). + +logs downloads output.log (a run's stdout+stderr, crash tracebacks included) to +workspace/wandb-logs//. --file picks another run file (config.yaml, +wandb-metadata.json, wandb-summary.json, requirements.txt). +""" + +import argparse +import os +import statistics + +import wandb + + +def run_maps(api, path): + runs = list(api.runs(path)) + return {r.id: r.name for r in runs}, {r.name: r.id for r in runs} + + +def to_id(id2name, name2id, sel, path): + if sel in id2name: + return sel + if sel in name2id: + return name2id[sel] + raise SystemExit(f"run not found in {path}: {sel!r}") + + +def series(run, key): + """ + Per-step {step: value} for a metric, via scan_history (returns every step). + """ + d = {} + for row in run.scan_history(keys=["_step", key]): + v = row.get(key) + if v is not None: + d[int(row["_step"])] = v + return d + + +def fmt(x): + return f"{x:+,.5f}" if abs(x) < 1000 else f"{x:+,.0f}" + + +def cmd_list(api, path): + runs = list(api.runs(path)) + print(f"{path}: {len(runs)} runs\n") + print(f"{'id':<12} {'state':<10} {'steps':>6} name") + keys = set() + for r in sorted(runs, key=lambda x: x.name): + print(f"{r.id:<12} {r.state:<10} {str(r.summary.get('_step')):>6} {r.name}") + keys |= {k for k in r.summary.keys() if not k.startswith("_")} + print("\nlogged metric keys:") + for k in sorted(keys): + print(f" {k}") + + +def cmd_compare(api, path, args): + id2name, name2id = run_maps(api, path) + ref_id = to_id(id2name, name2id, args.ref, path) + ref = series(api.run(f"{path}/{ref_id}"), args.metric) + vars_ = { + sel: series(api.run(f"{path}/{to_id(id2name, name2id, sel, path)}"), args.metric) + for sel in args.runs.split(",") + } + + steps = sorted(set(ref).intersection(*[set(v) for v in vars_.values()])) + if args.drop_first and steps: + steps = steps[1:] + if not steps: + raise SystemExit("no common steps with this metric") + fs = steps[-1] + + print( + f"metric: {args.metric} ref: {id2name[ref_id]} steps: {len(steps)} " + f"[{steps[0]}-{fs}]{' (dropped step 0)' if args.drop_first else ''}\n" + ) + + for sel, var in vars_.items(): + deltas = [var[s] - ref[s] for s in steps] + final = deltas[-1] + settling = statistics.mean(deltas[-min(5, len(deltas)) :]) + print(f"=== {sel} vs {id2name[ref_id]} ===") + print( + f" per-step delta : range [{fmt(min(deltas))}, {fmt(max(deltas))}] " + f"final(step {fs}) {fmt(final)} settling {fmt(settling)}\n" + ) + + +def cmd_logs(api, path, args): + id2name, name2id = run_maps(api, path) + for sel in args.runs.split(","): + r = api.run(f"{path}/{to_id(id2name, name2id, sel, path)}") + dest = f"workspace/wandb-logs/{r.name}" + os.makedirs(dest, exist_ok=True) + f = r.file(args.file).download(root=dest, replace=True) + n = sum(1 for _ in open(f.name)) + print(f"{r.name}: {f.name} ({os.path.getsize(f.name):,} bytes, {n} lines)") + + +def main(): + ap = argparse.ArgumentParser(usage=__doc__) + sub = ap.add_subparsers(dest="cmd", required=True) + + sub.add_parser("list").add_argument("path") + + sp = sub.add_parser("compare") + sp.add_argument("path") + sp.add_argument("--metric", required=True) + sp.add_argument("--ref", required=True) + sp.add_argument("--runs", required=True, help="comma-separated ids or names") + sp.add_argument("--drop-first", action="store_true") + + sp = sub.add_parser("logs") + sp.add_argument("path") + sp.add_argument("--runs", required=True, help="comma-separated ids or names") + sp.add_argument("--file", default="output.log", help="which wandb file to download") + + args = ap.parse_args() + api = wandb.Api() + if args.cmd == "list": + cmd_list(api, args.path) + elif args.cmd == "compare": + cmd_compare(api, args.path, args) + elif args.cmd == "logs": + cmd_logs(api, args.path, args) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/wandb-tracking/scripts/views.py b/.agents/skills/wandb-tracking/scripts/views.py new file mode 100644 index 0000000..4d77fa6 --- /dev/null +++ b/.agents/skills/wandb-tracking/scripts/views.py @@ -0,0 +1,232 @@ +""" +Create / update / list / delete wandb project views (the ?nw= saved +workspaces). + + python scripts/views.py list + python scripts/views.py show + python scripts/views.py set --show RUN[,RUN...] --name DISPLAYNAME + [--from-view NW_SLUG] [--update NW_SLUG] + python scripts/views.py delete + +Key facts (see references/views.md for the full anatomy): + - URL ?nw= maps to internal view name nw--v. + - A runset with empty filters means ALL runs are in the table; the selections + object narrows what's shown. With selections.root == 1, the tree lists the + HIDDEN (eye-closed) runs, so shown = all_runs - tree. + - set clones a template spec (an existing view, --from-view, or a built-in + fallback), rewrites selections.tree = all_ids - show_ids, and upserts. + Pass --update to modify an existing view in place (same URL). +""" + +import argparse +import json +import random +import string +import sys + +import wandb + +VIEWS_Q = """ +query Views($entityName: String!, $projectName: String!) { + project(name: $projectName, entityName: $entityName) { + allViews(viewType: "project-view") { + edges { node { id name displayName spec } } + } + } +} +""" + +UPSERT_M = """ +mutation Up($input: UpsertViewInput!) { + upsertView(input: $input) { view { id name displayName } inserted } +} +""" + +DELETE_M = """ +mutation Del($input: DeleteViewInput!) { deleteView(input: $input) { success } } +""" + +# Fallback spec (auto panels), used when the project has no view to clone from. +# tree and id are filled in by the set command. +FALLBACK_SPEC = { + "section": { + "name": "", + "version": 1, + "openRunSet": 0, + "openViz": True, + "runSets": [ + { + "id": "REPLACE_ID", + "runFeed": { + "version": 2, + "columnVisible": {"run:name": False}, + "columnPinned": {}, + "columnWidths": {}, + "columnOrder": [], + "pageSize": 20, + "onlyShowSelected": False, + "metricValences": {}, + }, + "search": {"query": ""}, + "searchHistory": [], + "name": "Run set", + "enabled": True, + "runNameTruncationType": "Middle", + "pinnedRunIds": [], + "filters": {"filterFormat": "filterV2", "filters": []}, + "grouping": [], + "sort": { + "keys": [{"key": {"section": "run", "name": "createdAt"}, "ascending": False}] + }, + "selections": {"root": 1, "bounds": [], "tree": []}, + "expandedRowAddresses": [], + } + ], + "panelBankConfig": { + "state": 1, + "settings": { + "showEmptySections": False, + "sortAlphabetically": False, + "defaultMoveToSectionName": "train", + }, + "sections": [], + }, + "panelBankSectionConfig": { + "__id__": "rps", + "name": "Report Panels", + "isOpen": False, + "sorted": 0, + "pinned": False, + "panels": [], + }, + "customRunColors": {}, + "customRunNames": {}, + "workspaceSettings": {"linePlot": {}}, + } +} + + +def slug(n): + return "".join(random.choices(string.ascii_lowercase + string.digits, k=n)) + + +def fetch_views(api, entity, project): + res = api._service_api.execute_graphql(VIEWS_Q, {"entityName": entity, "projectName": project}) + return [edge["node"] for edge in res["project"]["allViews"]["edges"]] + + +def shown_hidden(spec, id2name): + tree = set(spec["section"]["runSets"][0]["selections"]["tree"]) + shown = sorted(id2name.get(i, i) for i in set(id2name) - tree) + hidden = sorted(id2name.get(i, i) for i in tree) + return shown, hidden + + +def main(): + ap = argparse.ArgumentParser(usage=__doc__) + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("list").add_argument("path") + for c in ("show", "delete"): + sp = sub.add_parser(c) + sp.add_argument("path") + sp.add_argument("nw") + sp = sub.add_parser("set") + sp.add_argument("path") + sp.add_argument("--show", required=True, help="run ids/names to make visible") + sp.add_argument("--name", help="display name (required for new views)") + sp.add_argument("--from-view", help="nw slug to clone panels/layout from") + sp.add_argument("--update", help="nw slug to modify in place (keeps the URL)") + args = ap.parse_args() + + entity, project = args.path.split("/", 1) + api = wandb.Api() + id2name = {r.id: r.name for r in api.runs(args.path)} + + if args.cmd == "list": + for n in fetch_views(api, entity, project): + nw = ( + n["name"][3:-2] if n["name"].startswith("nw-") and n["name"].endswith("-v") else "?" + ) + print(f"- {n['displayName']!r} (nw={nw}, name={n['name']})") + try: + shown, hidden = shown_hidden(json.loads(n["spec"]), id2name) + print(f" shown : {shown}") + print(f" hidden: {hidden}") + except (KeyError, IndexError, json.JSONDecodeError): + print(" (no runset selection in spec)") + return + + views = fetch_views(api, entity, project) + + if args.cmd in ("show", "delete"): + node = next((v for v in views if v["name"] == f"nw-{args.nw}-v"), None) + if node is None: + raise SystemExit(f"view nw={args.nw} not found in {args.path}") + if args.cmd == "show": + shown, hidden = shown_hidden(json.loads(node["spec"]), id2name) + print(f"{node['displayName']!r} (nw={args.nw})") + print(f" shown : {shown}") + print(f" hidden: {hidden}") + else: + res = api._service_api.execute_graphql(DELETE_M, {"input": {"id": node["id"]}}) + print( + f"deleted {node['displayName']!r} (nw={args.nw}): success={res['deleteView']['success']}" + ) + return + + # set (create or update) + if not args.update and not args.name: + sys.exit("--name is required when creating a new view") + name2id = {v: k for k, v in id2name.items()} + show_ids = {sel if sel in id2name else name2id[sel] for sel in args.show.split(",")} + tree = sorted(set(id2name) - show_ids) + + template = None + if args.update: + template = next((v for v in views if v["name"] == f"nw-{args.update}-v"), None) + if template is None: + raise SystemExit(f"--update nw={args.update} not found") + elif args.from_view: + template = next((v for v in views if v["name"] == f"nw-{args.from_view}-v"), None) + if template is None: + raise SystemExit(f"--from-view nw={args.from_view} not found") + elif views: + template = views[0] + spec = json.loads(template["spec"]) if template else json.loads(json.dumps(FALLBACK_SPEC)) + spec["section"]["runSets"][0]["selections"] = {"root": 1, "bounds": [], "tree": tree} + + if args.update: + nw = args.update + inp = { + "id": template["id"], + "entityName": entity, + "projectName": project, + "type": "project-view", + "name": template["name"], + "displayName": args.name or template["displayName"], + "spec": json.dumps(spec), + } + else: + nw = slug(11) + spec["section"]["runSets"][0]["id"] = slug(9) + inp = { + "entityName": entity, + "projectName": project, + "type": "project-view", + "name": f"nw-{nw}-v", + "displayName": args.name, + "createdUsing": "WANDB_SDK", + "spec": json.dumps(spec), + } + + res = api._service_api.execute_graphql(UPSERT_M, {"input": inp}) + v = res["upsertView"]["view"] + print( + f"{'inserted' if res['upsertView']['inserted'] else 'updated'}: {v['displayName']!r} (nw={nw})" + ) + print(f"URL: https://wandb.ai/{args.path}?nw={nw}") + print(f"shown : {sorted(id2name[i] for i in show_ids)}") + + +if __name__ == "__main__": + main()