gen_config.py

#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-2-Clause
"""gen_config.py — reads board-config.xml, the single source of truth for
"which optional features are armed" and "what the guest's DTB must advertise
for them to do anything", and emits/applies both from one place:

  1. config.mk — a small Makefile fragment (VCPU2=0/1, VCPU1=0/1, HV_HDMI=..,
     HDMI_MODE_1080P=..) that the Makefile includes BEFORE its own `?=`
     defaults, so this file's values win without editing the Makefile.
  2. DTB edits (fdtput, idempotent) for every cpu@N node and virtio_mmio@...
     node board-config.xml says should exist given the current feature
     flags — the same manual recipe used all session for vinput/vcpu2, now
     driven from one file instead of five ad-hoc `fdtput` invocations typed
     by hand and easy to forget half of.

WHY: this session hit the exact "code says on, DTB doesn't advertise it"
mismatch twice (vcpu2's cpu@2 lived only in a separate, never-merged
bananapi-2cpu.dtb; vcpu1 shipped with no cpu@1 node at all). Both looked like
"armed and ready" from the C/Makefile side while being silently inert on
real hardware. This script makes that mismatch structurally impossible for
anything it manages: one file, one command, both artifacts updated together.

USAGE:
    python3 gen_config.py                 # apply to the live tftpboot DTB
    python3 gen_config.py --dtb PATH      # apply to a specific dtb instead
    python3 gen_config.py --dry-run       # print what would change, do nothing
    python3 gen_config.py --no-backup     # skip the automatic .pre-genconfig copy

Requires `dtc`/`fdtput` on PATH (already required by this project's own
existing DTB-editing docs) and Python's stdlib only.
"""
import argparse
import re
import subprocess
import sys
import xml.etree.ElementTree as ET
from pathlib import Path

HERE = Path(__file__).resolve().parent
DEFAULT_XML = HERE / "board-config.xml"
DEFAULT_DTB = Path("/opt/bzdos/tftpboot/bananapi-min.dtb")
# The /memory size the board ships with when no feature widens it: the low
# GiB, matching stage2.h's STAGE2_DRAM_SIZE default.
DEFAULT_MEMORY_SIZE = "0x40000000"
CONFIG_MK = HERE / "config.mk"

# SPI numbers must stay inside the documented free gap (docs/virtio-blk-dtb.md).
SPI_GAP = range(0x68, 0x74)

# Existing-hardware nodes no <feature dtb-nodes="..."> may ever list, checked
# structurally (not just by review) for the same reason wdogtrap.c's WDOG
# check is a hard first-return rather than a case a future edit could shuffle
# past. Each one is a real hazard already root-caused elsewhere in this tree:
#   - usb@1c19000 (MUSB/OTG) and usb@1c1a000/usb@1c1a400 (EHCI0/OHCI0) share
#     the OTG PHY (phys = <&usbphy 0>, same "usb0" lane as MUSB) with the
#     hypervisor's OWN debug/break-glass console (usbacm.c) -- see usbacm.h's
#     "KNOWN OPEN RISK". EHCI1/OHCI1 use the independent "usb1" lane instead
#     (phys = <&usbphy 1>) and are the pair guest_usb_host1 in
#     board-config.xml actually enables.
#   - watchdog@1c20ca0 is disabled BY DESIGN (PROGRESS.md 2026-07-15) -- the
#     board's only unattended recovery path. See CLAUDE.md.
#   - dma-controller@1c02000 is the SoC's general-purpose memory-to-memory
#     DMA engine: no SMMU gates it, and docs/dma-bypass-stage2.md documents
#     it as "provably unused" today specifically because every consumer that
#     could arm it is also disabled. dai@1c22c00 (the codec-i2s DAI, the
#     ONLY thing in the audio chain with a `dmas` property) is its one path
#     back to armed; leaving both off keeps that invariant true.
FORBIDDEN_DTB_NODES = {
    "/soc/usb@1c19000", "/soc/usb@1c1a000", "/soc/usb@1c1a400",
    "/soc/watchdog@1c20ca0",
    "/soc/dma-controller@1c02000", "/soc/dai@1c22c00",
}


def load_config(xml_path):
    tree = ET.parse(xml_path)
    root = tree.getroot()

    features = {}
    for f in root.find("features"):
        features[f.get("name")] = {
            "mkvar": f.get("mkvar"),
            "enabled": f.get("enabled", "false") == "true",
            "needs_dtb_cpu": f.get("needs-dtb-cpu"),
            "dtb_memory_size": f.get("dtb-memory-size"),
            # dtb-only: no C code anywhere is gated by this feature, only
            # what the guest's DTB advertises -- see board-config.xml's
            # guest_usb_host1-and-siblings comment. Skips the mkvar
            # requirement below and is never written to config.mk.
            "dtb_only": f.get("dtb-only", "false") == "true",
        }

    devices = []
    for d in root.find("devices"):
        devices.append({
            "name": d.get("name"),
            "base": d.get("base"),
            "spi": d.get("spi"),
            "device_id": d.get("device-id"),
            "always": d.get("always", "false") == "true",
            "enabled_by": d.get("enabled-by"),
            "no_dtb_node": d.get("no-dtb-node", "false") == "true",
        })

    soc_nodes = []
    soc_nodes_el = root.find("soc-nodes")
    if soc_nodes_el is not None:
        for n in soc_nodes_el:
            soc_nodes.append({
                "feature": n.get("feature"),
                "path": n.get("path"),
            })

    return features, devices, soc_nodes


def validate(features, devices, soc_nodes):
    """Cheap, load-bearing sanity checks before touching anything real."""
    errors = []

    seen_spi = {}
    for d in devices:
        if d["no_dtb_node"] or d["spi"] in (None, "none"):
            continue
        spi = int(d["spi"], 0)
        if spi not in SPI_GAP:
            errors.append(f"device {d['name']}: SPI {spi} (0x{spi:x}) is "
                          f"outside the documented free gap 0x68..0x73 — "
                          f"check docs/virtio-blk-dtb.md before using it")
        if spi in seen_spi:
            errors.append(f"device {d['name']}: SPI {spi} (0x{spi:x}) "
                           f"already used by device {seen_spi[spi]}")
        seen_spi[spi] = d["name"]

    for name, f in features.items():
        if not f["mkvar"] and not f["dtb_only"]:
            errors.append(f"feature {name}: no mkvar= attribute (and not "
                           f"dtb-only=\"true\") — gen_config.py doesn't know "
                           f"which Makefile variable this controls")

    for n in soc_nodes:
        if not n["path"] or not n["path"].startswith("/"):
            errors.append(f"soc-node feature={n['feature']}: path "
                           f"{n['path']!r} must be an absolute DTB path")
        if n["feature"] not in features:
            errors.append(f"soc-node path={n['path']}: feature "
                           f"{n['feature']!r} has no matching <feature> entry")
        if n["path"] in FORBIDDEN_DTB_NODES:
            errors.append(f"soc-node path={n['path']} (feature "
                           f"{n['feature']!r}): this node is in "
                           f"FORBIDDEN_DTB_NODES — see that constant's "
                           f"comment for which hazard it reopens, and do not "
                           f"remove the node from the set to get past this")

    if errors:
        for e in errors:
            print(f"[gen_config] ERROR: {e}", file=sys.stderr)
        sys.exit(1)


def emit_makefile_fragment(features, dry_run):
    lines = [
        "# GENERATED by gen_config.py from board-config.xml. Do not hand-edit —",
        "# your changes will be silently overwritten the next time someone runs",
        "# `python3 gen_config.py`. Edit board-config.xml instead.",
        "",
    ]
    for name, f in sorted(features.items()):
        if f["dtb_only"]:
            continue
        lines.append(f"{f['mkvar']} = {1 if f['enabled'] else 0}")
    content = "\n".join(lines) + "\n"

    if dry_run:
        print(f"[gen_config] would write {CONFIG_MK}:")
        print(content)
        return
    CONFIG_MK.write_text(content)
    print(f"[gen_config] wrote {CONFIG_MK}")


def dtb_to_dts_text(dtb_path):
    r = subprocess.run(["dtc", "-I", "dtb", "-O", "dts", str(dtb_path)],
                        capture_output=True, text=True)
    return r.stdout


def node_exists(dts_text, node_name):
    return re.search(rf"\b{re.escape(node_name)}\s*{{", dts_text) is not None


def parse_node_cells(dts_text, node_name, prop):
    """Pull a <...> cell-list property's values out of one node's block, as
    ints. Returns None if the node or property isn't found. Used to check an
    EXISTING node actually has the values it should, not just that it
    exists — see the fdtput() hex-vs-decimal bug this caught."""
    m = re.search(rf"\b{re.escape(node_name)}\s*{{(.*?)\n\t*}};", dts_text, re.S)
    if not m:
        return None
    pm = re.search(rf"\b{re.escape(prop)}\s*=\s*<([^>]*)>", m.group(1))
    if not pm:
        return None
    return [int(x, 16) for x in re.findall(r"0x[0-9a-fA-F]+", pm.group(1))]


def fdtput(dtb_path, node, prop_type, prop, *values, dry_run=False):
    # fdtput's "-t x" parses each argument string AS HEX DIGITS, not as a
    # decimal string of a value already computed in Python. Passing str(v)
    # for an int v (e.g. str(0x0A004000) == "167780352") makes fdtput read
    # THAT decimal-looking string as hex, silently corrupting reg/interrupts
    # for any multi-digit value — bit vblk_sd's base/SPI on first use (cpu_id
    # values 1/2/3 happened to survive since a single digit is the same in
    # both bases). Hex-format ints explicitly for "x" so the two never mix.
    if prop_type == "x":
        vals = [v if isinstance(v, str) else format(v, "x") for v in values]
    else:
        vals = [str(v) for v in values]
    cmd = ["fdtput", "-p", "-t", prop_type, str(dtb_path), node, prop, *vals]
    if dry_run:
        print("[gen_config] would run:", " ".join(cmd))
        return
    subprocess.run(cmd, check=True)


def ensure_cpu_node(dtb_path, dts_text, cpu_id, dry_run):
    node = f"cpu@{cpu_id}"
    path = f"/cpus/{node}"
    if node_exists(dts_text, node):
        print(f"[gen_config] {node}: already present, leaving as-is")
        return
    print(f"[gen_config] {node}: adding (minimal, mirrors cpu@0's essential fields)")
    fdtput(dtb_path, path, "s", "compatible", "arm,cortex-a53", dry_run=dry_run)
    fdtput(dtb_path, path, "s", "device_type", "cpu", dry_run=dry_run)
    fdtput(dtb_path, path, "x", "reg", cpu_id, dry_run=dry_run)
    fdtput(dtb_path, path, "s", "enable-method", "psci", dry_run=dry_run)


def ensure_virtio_node(dtb_path, dts_text, dev, dry_run):
    addr_hex = int(dev["base"], 0)
    node_name = f"virtio_mmio@{addr_hex:x}"
    path = f"/soc/{node_name}"
    spi = int(dev["spi"], 0)
    expected_reg = [addr_hex, 0x200]
    expected_irq = [0x0, spi, 0x1]
    if node_exists(dts_text, node_name):
        reg = parse_node_cells(dts_text, node_name, "reg")
        irq = parse_node_cells(dts_text, node_name, "interrupts")
        if reg == expected_reg and irq == expected_irq:
            print(f"[gen_config] {node_name} ({dev['name']}): already present and correct, leaving as-is")
            return
        print(f"[gen_config] {node_name} ({dev['name']}): present but WRONG "
              f"(reg={reg}, interrupts={irq}, expected reg={expected_reg} "
              f"interrupts={expected_irq}) — correcting")
    else:
        print(f"[gen_config] {node_name} ({dev['name']}): adding")
    fdtput(dtb_path, path, "s", "compatible", "virtio,mmio", dry_run=dry_run)
    fdtput(dtb_path, path, "x", "reg", addr_hex, 0x200, dry_run=dry_run)
    fdtput(dtb_path, path, "x", "interrupts", 0x0, spi, 0x1, dry_run=dry_run)
    fdtput(dtb_path, path, "x", "interrupt-parent", 0x1, dry_run=dry_run)
    fdtput(dtb_path, path, "s", "status", "okay", dry_run=dry_run)


def reorder_virtio_nodes(dtb_path, dry_run):
    """FreeBSD assigns vtbdN/vtnetN unit numbers in DTB CHILD-LIST order,
    NOT by each node's 'reg' address — confirmed live 2026-08-24: adding
    vblk_sd via fdtput -p put its virtio_mmio@a004000 node BEFORE the three
    pre-existing ones (fdtput -p inserts a new node at the head of its
    parent's child list), which silently swapped vtbd0 (root, hardcoded as
    'ufs:/dev/vtbd0p3' in kload.c's kenv and in bzd_board.py's
    ROOT_MOUNTFROM) onto the SD card instead of the eMMC, and the guest sat
    at the mountroot prompt until fixed by hand.
    Leaving node order as "whatever fdtput happened to insert" is a landmine
    for every future device added through board-config.xml, so this makes
    ascending-by-address order a structural invariant re-enforced on every
    run, not a one-off manual fix. Requires the virtio_mmio@ nodes to be
    CONTIGUOUS siblings (true today: they're the only things gen_config.py
    ever inserts into /soc, and fdtput -p always inserts at the same spot
    right after /soc's own properties) — if a future change breaks that
    assumption, this only warns and leaves the DTB untouched rather than
    risk reassembling the tree wrong."""
    dts_text = dtb_to_dts_text(dtb_path)
    blocks = list(re.finditer(r"[ \t]*virtio_mmio@([0-9a-fA-F]+)\s*\{.*?\n[ \t]*\};\n?",
                               dts_text, re.S))
    if len(blocks) < 2:
        return
    for a, b in zip(blocks, blocks[1:]):
        if dts_text[a.end():b.start()].strip():
            print("[gen_config] NOTE: virtio_mmio nodes are no longer "
                  "contiguous siblings — skipping the address-order check "
                  "(reorder_virtio_nodes()'s contiguity assumption no "
                  "longer holds, fix it before trusting vtbdN==this-device "
                  "assumptions anywhere)")
            return

    addrs = [int(b.group(1), 16) for b in blocks]
    if addrs == sorted(addrs):
        print("[gen_config] virtio_mmio nodes already in address order, leaving as-is")
        return

    print(f"[gen_config] virtio_mmio nodes out of order ({[hex(a) for a in addrs]}) "
          f"— this reshuffles vtbdN/vtnetN unit numbers on the guest — resorting")
    if dry_run:
        print("[gen_config] would resort and recompile the DTB")
        return

    ordered = sorted(blocks, key=lambda b: int(b.group(1), 16))
    start, end = blocks[0].start(), blocks[-1].end()
    new_text = dts_text[:start] + "".join(b.group(0) for b in ordered) + dts_text[end:]

    tmp_dts = dtb_path.with_suffix(".gen_config_reorder.dts")
    tmp_dts.write_text(new_text)
    try:
        subprocess.run(["dtc", "-I", "dts", "-O", "dtb", "-o", str(dtb_path), str(tmp_dts)],
                        check=True, capture_output=True, text=True)
    finally:
        tmp_dts.unlink()
    print("[gen_config] DTB recompiled with virtio_mmio nodes in address order")


def ensure_memory_size(dtb_path, dts_text, size_hex, dry_run):
    """Keep the DTB's /memory node in step with STAGE2_DRAM_SIZE.

    The guest learns how much RAM it has from this node and nothing rewrites it
    at load time -- kload.c builds a kenv string but never touches /memory. So a
    build with GUEST_DRAM_2G and a 1 GiB /memory node gives the guest a 2 GiB
    stage-2 window it will never allocate out of, and the reverse gives it a
    node promising memory stage-2 will fault on. Both halves are set from one
    switch here for exactly the reason the cpu@N handling above exists."""
    want = int(size_hex, 16)
    cur = fdtget_ints(dtb_path, "/memory", "reg")
    if cur is not None and len(cur) >= 2 and cur[1] == want:
        print(f"[gen_config] /memory: size already {want:#x}, leaving as-is")
        return
    have = f"{cur[1]:#x}" if cur and len(cur) >= 2 else "unreadable"
    # MiB, not GiB: a non-power-of-two size like 0x78000000 (1920 MiB) printed
    # as "want >> 30 GiB" reads as "1 GiB", which is exactly the sort of quietly
    # wrong log line that costs someone an hour later.
    print(f"[gen_config] /memory: size {have} -> {want:#x} "
          f"({want >> 20} MiB), base left at 0x40000000")
    fdtput(dtb_path, "/memory", "x", "reg", 0x40000000, want, dry_run=dry_run)


def fdtget_ints(dtb_path, node, prop):
    r = subprocess.run(["fdtget", str(dtb_path), node, prop],
                       capture_output=True, text=True)
    if r.returncode != 0:
        return None
    try:
        return [int(x) for x in r.stdout.split()]
    except ValueError:
        return None


def fdtget_str(dtb_path, node, prop):
    r = subprocess.run(["fdtget", str(dtb_path), node, prop],
                        capture_output=True, text=True)
    if r.returncode != 0:
        return None
    return r.stdout.strip()


def ensure_node_status(dtb_path, node_path, want_enabled, dry_run):
    """Flip ONE existing silicon node's `status` between "okay"/"disabled" —
    see board-config.xml's <soc-nodes>. Never adds or removes a node, only
    the property that gates whether FreeBSD's OFW device-enumeration probes
    it at all. A DTB node with no `status` property is implicitly "okay" per
    the devicetree spec, so an unreadable/missing property is treated as
    "okay", not as "needs no action" — this is the same both-directions
    self-healing reconcile_drift() does for cpu@N nodes, applied here to a
    property instead of a whole node."""
    want = "okay" if want_enabled else "disabled"
    cur = fdtget_str(dtb_path, node_path, "status")
    cur_effective = cur if cur is not None else "okay"
    if cur_effective == want:
        print(f"[gen_config] {node_path}: status already {want}, leaving as-is")
        return
    print(f"[gen_config] {node_path}: status {cur_effective} -> {want}")
    fdtput(dtb_path, node_path, "s", "status", want, dry_run=dry_run)


def remove_cpu_node(dtb_path, cpu_id, dry_run):
    node = f"cpu@{cpu_id}"
    path = f"/cpus/{node}"
    print(f"[gen_config] {node}: REMOVING (its feature is disabled — a node "
          f"without the build flag is the same mismatch as a flag without "
          f"the node)")
    if dry_run:
        return
    subprocess.run(["fdtput", "-r", str(dtb_path), path], check=True)


def reconcile_drift(features, dts_text, dtb_path, dry_run):
    """Both directions of the mismatch this script exists to prevent.

    An earlier version of this function only PRINTED a note here, on the
    reasoning that a leftover node might be "deliberate for a follow-up boot".
    That reasoning cost real board time on 2026-08-27: vcpu2 was armed, the
    guest stalled, and disarming it again left cpu@2 sitting in the DTB with
    VCPU2=0 — so FreeBSD would enumerate a third core, issue PSCI CPU_ON for
    affinity 2, and EL2 would refuse it. That is precisely the half-configured
    state the header of this file and board-config.xml both warn about, in the
    one direction the script had declined to actually fix. A note you have to
    notice is not a safeguard.

    So: remove it. The DTB is a generated artifact here, a backup is written
    above, and "I want that node" is spelled by enabling the feature."""
    for name, f in features.items():
        if f["needs_dtb_cpu"] and not f["enabled"]:
            node = f"cpu@{f['needs_dtb_cpu']}"
            if node_exists(dts_text, node):
                remove_cpu_node(dtb_path, f["needs_dtb_cpu"], dry_run)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--xml", default=str(DEFAULT_XML))
    ap.add_argument("--dtb", default=str(DEFAULT_DTB))
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--no-backup", action="store_true")
    args = ap.parse_args()

    features, devices, soc_nodes = load_config(args.xml)
    validate(features, devices, soc_nodes)

    emit_makefile_fragment(features, args.dry_run)

    dtb_path = Path(args.dtb)
    if not dtb_path.exists():
        print(f"[gen_config] {dtb_path} not found — skipping DTB step "
              f"(config.mk was still written)", file=sys.stderr)
        return

    if not args.dry_run and not args.no_backup:
        backup = dtb_path.with_suffix(dtb_path.suffix + ".pre-genconfig")
        if not backup.exists():
            backup.write_bytes(dtb_path.read_bytes())
            print(f"[gen_config] backup: {backup}")

    dts_text = dtb_to_dts_text(dtb_path)
    reconcile_drift(features, dts_text, dtb_path, args.dry_run)
    if not args.dry_run:
        dts_text = dtb_to_dts_text(dtb_path)   # a removal invalidates it

    for name, f in features.items():
        if f["enabled"] and f["needs_dtb_cpu"]:
            ensure_cpu_node(dtb_path, dts_text, f["needs_dtb_cpu"], args.dry_run)

    # /memory tracks whichever feature declares a dtb-memory-size -- and, just as
    # importantly, gets put BACK when none is enabled. Leaving it wide after the
    # feature is switched off is the same one-directional bug this script had for
    # a stale cpu@N node, and it is worse here: a /memory node promising a GiB
    # that stage-2 no longer maps means the guest faults the moment it allocates
    # up there, which is exactly how it presented (translation fault, level 1,
    # IPA 0xBFFF0000) when the two halves were briefly out of step.
    want = None
    for name, f in features.items():
        if f["enabled"] and f["dtb_memory_size"]:
            want = f["dtb_memory_size"]
    ensure_memory_size(dtb_path, dts_text, want or DEFAULT_MEMORY_SIZE,
                       args.dry_run)

    # NOTE on ORDER, observed the same day: ensure_cpu_node() adds nodes with
    # `fdtput -p`, which PREPENDS. Arming vcpu2 therefore produced /cpus in the
    # order `cpu@2 cpu@1 cpu@0`, and FreeBSD duly enumerated "CPU 1 ...
    # affinity: 2" and "CPU 2 ... affinity: 1" -- its logical numbering is DTB
    # order, not MPIDR. Harmless for bring-up but genuinely confusing while
    # reading a console log, and the exact same prepend footgun that once
    # silently swapped vtbd0/vtbd1 (see the virtio ordering self-heal below,
    # which exists for that reason). Not self-healed for cpu nodes yet.

    for d in devices:
        if d["no_dtb_node"]:
            continue
        wanted = d["always"] or (d["enabled_by"] and features.get(d["enabled_by"], {}).get("enabled"))
        if wanted:
            ensure_virtio_node(dtb_path, dts_text, d, args.dry_run)

    reorder_virtio_nodes(dtb_path, args.dry_run)

    # Real silicon nodes gated on a dtb-only feature (board-config.xml's
    # <soc-nodes>) — always reconciled, both directions, same as everything
    # above: a node left "okay" after its feature is switched off again is
    # the same drift this whole script exists to prevent.
    for n in soc_nodes:
        f = features.get(n["feature"], {})
        ensure_node_status(dtb_path, n["path"], f.get("enabled", False), args.dry_run)

    print("[gen_config] done")


if __name__ == "__main__":
    main()