From a4e9d0a08058f86c8114cf647ffcbb7535fea313 Mon Sep 17 00:00:00 2001 From: bryan Date: Tue, 14 Jul 2026 15:43:10 -0700 Subject: [PATCH 1/2] ci: add report-only corresponding-source check (inbox#283 / inbox#53) For every copyleft package (classified from license_spdx via prefix match on expression tokens), verify its exact source archive is durably present on the public mirror - the corresponding-source offer that distributing copyleft binaries from the cache obligates. Presence is probed with unauthenticated HTTPS HEADs against the public bucket (fixed host, charset-validated paths, no secrets); exactness rides on the build's own sha256 pin. Doubled slashes are normalized to match minimal's gs:// fetcher (graphviz's build.ncl has a literal // that builds fine). First full-catalog run: 127 copyleft packages -> 89 mirrored / 0 mirror-missing / 38 unmirrored (ffmpeg, ghostscript+grafana both AGPL, jdk, gcc-arm-none-eabi, libvips, x264/x265, ...) - the concrete #283 work list. Runs on PRs (changed packages), dispatch, and a weekly schedule: mirror objects can vanish independently of any PR, and that silent drift is the main thing to catch. Seventh sibling of the report-only check family; MUST NOT be required. Co-Authored-By: Claude Fable 5 --- .../scripts/corresponding_source_report.py | 217 ++++++++++++++++++ .github/workflows/corresponding-source.yml | 74 ++++++ 2 files changed, 291 insertions(+) create mode 100644 .github/scripts/corresponding_source_report.py create mode 100644 .github/workflows/corresponding-source.yml diff --git a/.github/scripts/corresponding_source_report.py b/.github/scripts/corresponding_source_report.py new file mode 100644 index 00000000..2b51ecc9 --- /dev/null +++ b/.github/scripts/corresponding_source_report.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Report-only corresponding-source check (gominimal/inbox#283, #53). + +For every COPYLEFT package (classified from its license_spdx expression), verify +that the exact source archive it is built from is durably available on the +public mirror (gs://minimal-staging-archives). Distributing a copyleft binary +from the cache carries a corresponding-source obligation; the mirror is our +mechanism for meeting it — so a copyleft package whose source is unmirrored +(fetched straight from GitHub/upstream) or whose mirror object has gone missing +is a compliance gap. Exactness rides on the build's own sha256 pin: if the +mirrored object drifted, builds would already fail, so presence == exactness. + +Categories per copyleft package: + ✅ mirrored — every source URL is the mirror and the object exists + ❌ mirror-miss — a mirror URL whose object is GONE (drift/GC: the worst case) + ⚠️ unmirrored — fetches upstream directly (GitHub, dl.grafana.com, …); a + retention/mirroring action is needed for the offer to hold + ❓ unresolved — URL interpolation couldn't be resolved statically + +Presence is probed with unauthenticated HTTPS HEADs against the public bucket +(https://storage.googleapis.com/minimal-staging-archives/) — no gcloud, +no secrets. Only that fixed host is ever contacted, and only with +charset-validated paths taken from tracked build.ncl (no SSRF surface). + +REPORT-ONLY: exits 0 for any policy verdict; non-zero only on a programming +error (downgraded to a warning by the workflow). MUST NOT be a required check. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys + +_LICENSE_RE = re.compile(r'^\s*(?:attrs\.)?license_spdx\s*=\s*"([^"]*)"', re.M) +_URL_RE = re.compile(r'^\s*url\s*=\s*"([^"]+)"', re.M) +_LET_RE = re.compile(r'^\s*let\s+([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"]*)"\s+in\s*$', re.M) +_INTERP_RE = re.compile(r"%\{([A-Za-z_][A-Za-z0-9_-]*)\}") +_MIRROR = "gs://minimal-staging-archives/" +_MIRROR_HTTP = "https://storage.googleapis.com/minimal-staging-archives/" +_SAFE_PATH = re.compile(r"^[A-Za-z0-9._/+~-]+$") + +# SPDX id prefixes that carry source obligations on binary distribution. +# Strong and weak/file-level copyleft both create a corresponding-source duty +# for the covered code, so both are in scope. Prefix-matched against expression +# tokens (GPL-2.0-only, LGPL-2.1-or-later, AGPL-3.0-only, MPL-2.0, ...). +_COPYLEFT_PREFIXES = ( + "GPL-", "AGPL-", "LGPL-", "SSPL-", "MPL-", "EPL-", "CDDL-", "EUPL-", + "CeCILL", "OSL-", "MS-RL", "CPL-", "IPL-", "RPL-", "CC-BY-SA-", +) +_TOKEN_SPLIT = re.compile(r"[()\s]+") + + +def copyleft_tokens(expr: str) -> list[str]: + toks = [t for t in _TOKEN_SPLIT.split(expr) if t and t not in ("AND", "OR", "WITH")] + return [t for t in toks if t.startswith(_COPYLEFT_PREFIXES)] + + +def resolve_interpolations(url: str, lets: dict[str, str]) -> str: + # let-bindings can reference each other (dl_base uses %{version}); a few + # passes of textual substitution resolves the practical cases. + for _ in range(5): + expanded = _INTERP_RE.sub(lambda m: lets.get(m.group(1), m.group(0)), url) + if expanded == url: + break + url = expanded + return url + + +def head_mirror_object(path: str) -> bool | None: + """True/False = object present/absent on the public mirror; None = probe + failed (network/5xx) — reported as unknown rather than missing.""" + if not _SAFE_PATH.match(path): + return None + try: + p = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + "--head", "--proto", "=https", "--max-time", "20", + _MIRROR_HTTP + path], + capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + return None + code = p.stdout.strip() + if code == "200": + return True + if code in ("403", "404"): # public bucket: absent objects read as 403/404 + return False + return None + + +def md_cell(s) -> str: + return (str(s).replace("\\", "\\\\").replace("|", "\\|") + .replace("`", "\\`").replace("\r", " ").replace("\n", " ")) + + +def emit(markdown: str) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a") as f: + f.write(markdown) + else: + print("\n" + markdown) + + +def main() -> int: + ap = argparse.ArgumentParser() + grp = ap.add_mutually_exclusive_group(required=True) + grp.add_argument("--all", action="store_true") + grp.add_argument("--packages-file") + args = ap.parse_args() + + if args.all: + try: + names = sorted(d for d in os.listdir("packages") + if os.path.isfile(os.path.join("packages", d, "build.ncl"))) + except OSError: + names = [] + else: + try: + with open(args.packages_file) as f: + names = [ln.strip() for ln in f if ln.strip()] + except OSError as e: + emit("## corresponding-source report (informational, non-blocking)\n\n" + f"> Could not read packages file ({type(e).__name__}); skipping.\n") + return 0 + + rows = [] + counts = {"mirrored": 0, "mirror-miss": 0, "unmirrored": 0, "unresolved": 0, "probe-unknown": 0} + for name in names: + try: + with open(os.path.join("packages", name, "build.ncl"), encoding="utf-8") as f: + src = f.read() + except OSError: + continue + m = _LICENSE_RE.search(src) + if not m: + continue + cl = copyleft_tokens(m.group(1)) + if not cl: + continue + + lets = dict(_LET_RE.findall(src)) + urls = [resolve_interpolations(u, lets) for u in _URL_RE.findall(src)] + # Local `file =` deps and empty specs: a copyleft package with no URLs + # builds from in-repo sources; the repo itself is the source offer. + verdicts, details = [], [] + for u in urls: + if "%{" in u: + verdicts.append("unresolved") + details.append(f"unresolved interpolation: {u}") + elif u.startswith(_MIRROR): + # minimal's gs:// fetcher tolerates doubled slashes (graphviz's + # build.ncl has a literal `//` and builds fine) — normalize so + # the probe matches fetcher behavior instead of phantom-missing. + path = re.sub(r"/{2,}", "/", u[len(_MIRROR):].lstrip("/")) + present = head_mirror_object(path) + if present is True: + verdicts.append("mirrored") + elif present is False: + verdicts.append("mirror-miss") + details.append(f"MISSING from mirror: {u}") + else: + verdicts.append("probe-unknown") + details.append(f"probe failed: {u}") + else: + verdicts.append("unmirrored") + details.append(f"upstream-only: {u}") + + if not urls: + continue # nothing fetched -> nothing to mirror + # Package verdict = worst URL verdict. + order = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored"] + worst = min(verdicts, key=order.index) + counts[worst] += 1 + icon = {"mirrored": "✅", "mirror-miss": "❌", "unmirrored": "⚠️", + "unresolved": "❓", "probe-unknown": "❓"}[worst] + licenses = " ".join(sorted(set(cl))) + rows.append((name, licenses, f"{icon} {worst}", "; ".join(details) or "—")) + print(f"{name} [{licenses}] -> {worst}" + (f" ({'; '.join(details)})" if details else "")) + + sev = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored"] + rows.sort(key=lambda r: (sev.index(r[2].split()[1]), r[0])) + + out = [ + "## corresponding-source report (informational, non-blocking)", "", + "For every **copyleft** package (GPL/LGPL/AGPL/MPL/…, classified from " + "`license_spdx`), is the exact source archive it is built from durably " + "available on the public mirror? Distribution of a copyleft binary from " + "the cache carries a corresponding-source obligation (gominimal/inbox#283 " + "/ #53); the mirror is the offer. Exactness rides on the build's sha256 " + "pin — presence is what can drift.", "", + f"**{counts['mirrored']} mirrored ✅ · {counts['mirror-miss']} mirror-MISSING ❌ · " + f"{counts['unmirrored']} unmirrored ⚠️ · " + f"{counts['unresolved'] + counts['probe-unknown']} unresolved/unknown ❓**", "", + ] + flagged = [r for r in rows if not r[2].endswith("mirrored")] or rows[:0] + shown = rows if not args.all else (flagged if flagged else []) + if shown: + out += ["| package | copyleft license(s) | verdict | detail |", + "| --- | --- | --- | --- |"] + out += ["| " + " | ".join(md_cell(c) for c in r) + " |" for r in shown] + elif args.all: + out.append("✅ Every copyleft package's source is mirrored and present.") + out += ["", + "> Report-only; never blocks and is excluded from required checks. " + "⚠️ unmirrored = fetched straight from upstream — the mirror holds " + "no copy, so the source offer depends on a third party. ❌ = a " + "mirror object we point at is gone (the drift this check exists to " + "catch)."] + emit("\n".join(out) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/corresponding-source.yml b/.github/workflows/corresponding-source.yml new file mode 100644 index 00000000..2040e11a --- /dev/null +++ b/.github/workflows/corresponding-source.yml @@ -0,0 +1,74 @@ +name: corresponding-source + +# REPORT-ONLY, NON-BLOCKING. For every copyleft package (classified from +# license_spdx), verify its exact source archive is durably present on the +# public mirror (gs://minimal-staging-archives) — the corresponding-source +# offer that distributing copyleft binaries from the cache obligates +# (gominimal/inbox#283 / #53). MUST NOT be a branch-protection required check. +# Seventh sibling of the report-only check family. +# +# Network: unauthenticated HTTPS HEADs against the public mirror bucket ONLY +# (fixed host; charset-validated paths from tracked build.ncl). No secrets. +# The weekly schedule exists because mirror objects can drift/vanish +# independently of any PR — that silent drift is the main thing to catch. + +on: + workflow_dispatch: {} + schedule: + - cron: "43 7 * * 1" # weekly drift sweep over the full catalog + pull_request: + branches: ["main"] + +permissions: + contents: read + +concurrency: + group: corresponding-source-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + report: + runs-on: ubuntu-latest + steps: + # Pinned to a SHA (matches the sibling report-only workflows). + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 # base..head reachable for the changed-file diff + + - name: Package set + id: pkgset + env: + # SHAs via env per repo convention (zizmor template-injection). + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT: ${{ github.event_name }} + run: | + set -uo pipefail + if [ "$EVENT" = "pull_request" ]; then + git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- 'packages/*/build.ncl' \ + | cut -d/ -f2 | sort -u > pkgs.txt || : > pkgs.txt + echo "mode=file" >> "$GITHUB_OUTPUT" + echo "count=$(wc -l < pkgs.txt | tr -d ' ')" >> "$GITHUB_OUTPUT" + else + : > pkgs.txt # --all sweeps the whole catalog + echo "mode=all" >> "$GITHUB_OUTPUT" + echo "count=1" >> "$GITHUB_OUTPUT" + fi + + - name: Corresponding-source report + if: steps.pkgset.outputs.count != '0' + run: | + set -uo pipefail + # Report-only: a genuine error is downgraded to a warning so the + # check never reds. + if [ "${{ steps.pkgset.outputs.mode }}" = "file" ]; then + python3 .github/scripts/corresponding_source_report.py --packages-file pkgs.txt \ + || echo "::warning::corresponding-source report hit an internal error; see logs (non-blocking)" + else + python3 .github/scripts/corresponding_source_report.py --all \ + || echo "::warning::corresponding-source report hit an internal error; see logs (non-blocking)" + fi + + - name: No package changes + if: steps.pkgset.outputs.count == '0' + run: echo "No packages/*/build.ncl changed in this PR — nothing to report." >> "$GITHUB_STEP_SUMMARY" From 8bfb8087e5e98a0f922f0bf1af23ea1f8b60e71f Mon Sep 17 00:00:00 2001 From: bryan Date: Tue, 14 Jul 2026 16:40:35 -0700 Subject: [PATCH 2/2] corresponding-source: recognize out-of-band mirrored sources + probe fixes The inbox#283 backfill uploads archives to the mirror WITHOUT touching build.ncl (repointing changes spec_hashes -> rebuild cascades; URLs get repointed lazily at natural version bumps). Teach the check to probe the conventional mirror path (github -> owner/repo/basename, else flat basename) for upstream-URL packages and report 'mirrored (out-of-band)' instead of falsely flagging them unmirrored. Probe fixes caught by the backfill itself: underscore was missing from the path charset (jdk's OpenJDK21U-jdk_x64_... failed validation before probing), and a failed probe in the out-of-band branch collapsed into 'unmirrored' instead of 'probe-unknown'. Full-catalog state after backfill: 127/127 copyleft packages mirrored (89 by URL, 38 out-of-band), 0 missing, 0 unmirrored. Co-Authored-By: Claude Fable 5 --- .../scripts/corresponding_source_report.py | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/scripts/corresponding_source_report.py b/.github/scripts/corresponding_source_report.py index 2b51ecc9..fa59a012 100644 --- a/.github/scripts/corresponding_source_report.py +++ b/.github/scripts/corresponding_source_report.py @@ -13,8 +13,11 @@ Categories per copyleft package: ✅ mirrored — every source URL is the mirror and the object exists ❌ mirror-miss — a mirror URL whose object is GONE (drift/GC: the worst case) - ⚠️ unmirrored — fetches upstream directly (GitHub, dl.grafana.com, …); a - retention/mirroring action is needed for the offer to hold + ✅ mirrored-oob — fetches upstream directly, but the archive IS present on + the mirror at the conventional path (mirrored out-of-band; + repoint the URL at the next natural version bump) + ⚠️ unmirrored — fetches upstream directly AND the mirror holds no copy; a + mirroring action is needed for the offer to hold ❓ unresolved — URL interpolation couldn't be resolved statically Presence is probed with unauthenticated HTTPS HEADs against the public bucket @@ -40,7 +43,7 @@ _INTERP_RE = re.compile(r"%\{([A-Za-z_][A-Za-z0-9_-]*)\}") _MIRROR = "gs://minimal-staging-archives/" _MIRROR_HTTP = "https://storage.googleapis.com/minimal-staging-archives/" -_SAFE_PATH = re.compile(r"^[A-Za-z0-9._/+~-]+$") +_SAFE_PATH = re.compile(r"^[A-Za-z0-9_./+~-]+$") # SPDX id prefixes that carry source obligations on binary distribution. # Strong and weak/file-level copyleft both create a corresponding-source duty @@ -69,6 +72,15 @@ def resolve_interpolations(url: str, lets: dict[str, str]) -> str: return url +def conventional_dest(url: str) -> str: + """Where the mirror convention would place this upstream URL: GitHub -> + //, other hosts -> flat (matches the + existing 289 mirrored sources and the inbox#283 backfill).""" + m = re.match(r"https://github\.com/([^/]+)/([^/]+)/", url) + base = url.split("?")[0].rstrip("/").split("/")[-1] + return f"{m.group(1)}/{m.group(2)}/{base}" if m else base + + def head_mirror_object(path: str) -> bool | None: """True/False = object present/absent on the public mirror; None = probe failed (network/5xx) — reported as unknown rather than missing.""" @@ -127,7 +139,7 @@ def main() -> int: return 0 rows = [] - counts = {"mirrored": 0, "mirror-miss": 0, "unmirrored": 0, "unresolved": 0, "probe-unknown": 0} + counts = {"mirrored": 0, "mirrored-oob": 0, "mirror-miss": 0, "unmirrored": 0, "unresolved": 0, "probe-unknown": 0} for name in names: try: with open(os.path.join("packages", name, "build.ncl"), encoding="utf-8") as f: @@ -164,6 +176,18 @@ def main() -> int: else: verdicts.append("probe-unknown") details.append(f"probe failed: {u}") + elif u.startswith("https://"): + dest = conventional_dest(u) + present = head_mirror_object(dest) + if present is True: + verdicts.append("mirrored-oob") + details.append(f"mirror holds {dest}; build.ncl still fetches upstream") + elif present is False: + verdicts.append("unmirrored") + details.append(f"upstream-only: {u}") + else: + verdicts.append("probe-unknown") + details.append(f"probe failed: {dest}") else: verdicts.append("unmirrored") details.append(f"upstream-only: {u}") @@ -171,16 +195,16 @@ def main() -> int: if not urls: continue # nothing fetched -> nothing to mirror # Package verdict = worst URL verdict. - order = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored"] + order = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored-oob", "mirrored"] worst = min(verdicts, key=order.index) counts[worst] += 1 - icon = {"mirrored": "✅", "mirror-miss": "❌", "unmirrored": "⚠️", - "unresolved": "❓", "probe-unknown": "❓"}[worst] + icon = {"mirrored": "✅", "mirrored-oob": "✅", "mirror-miss": "❌", + "unmirrored": "⚠️", "unresolved": "❓", "probe-unknown": "❓"}[worst] licenses = " ".join(sorted(set(cl))) rows.append((name, licenses, f"{icon} {worst}", "; ".join(details) or "—")) print(f"{name} [{licenses}] -> {worst}" + (f" ({'; '.join(details)})" if details else "")) - sev = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored"] + sev = ["mirror-miss", "unmirrored", "unresolved", "probe-unknown", "mirrored-oob", "mirrored"] rows.sort(key=lambda r: (sev.index(r[2].split()[1]), r[0])) out = [ @@ -191,11 +215,13 @@ def main() -> int: "the cache carries a corresponding-source obligation (gominimal/inbox#283 " "/ #53); the mirror is the offer. Exactness rides on the build's sha256 " "pin — presence is what can drift.", "", - f"**{counts['mirrored']} mirrored ✅ · {counts['mirror-miss']} mirror-MISSING ❌ · " + f"**{counts['mirrored'] + counts['mirrored-oob']} mirrored ✅ " + f"(of which {counts['mirrored-oob']} out-of-band) · " + f"{counts['mirror-miss']} mirror-MISSING ❌ · " f"{counts['unmirrored']} unmirrored ⚠️ · " f"{counts['unresolved'] + counts['probe-unknown']} unresolved/unknown ❓**", "", ] - flagged = [r for r in rows if not r[2].endswith("mirrored")] or rows[:0] + flagged = [r for r in rows if r[2].split()[1] not in ("mirrored", "mirrored-oob")] or rows[:0] shown = rows if not args.all else (flagged if flagged else []) if shown: out += ["| package | copyleft license(s) | verdict | detail |",