From c7ca0191c06299f4be088dbe71d8b6d255d27b65 Mon Sep 17 00:00:00 2001 From: DTTerastar Date: Tue, 21 Jul 2026 20:26:30 -0400 Subject: [PATCH] qa: add manifest-check tool for #2225 (failed to download manifest) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the manifest endpoint probe into this repo — the one that actually serves the /releases/{version}.json?flavor={flavor} endpoints via functions/releases/[[path]].ts. Tool: tools/manifest_check/check_manifest.py (stdlib-only). GETs each manifest, validates ESP Web Tools schema, HEADs each binary URL. Part paths are resolved relative to the manifest URL (matching the esp-web-tools client). Retries on 5xx and 429 (GitHub rate-limit). CI: .github/workflows/manifest-check.yml runs unit tests + a live production probe on PR / push / hourly cron / dispatch. The production probe currently FAILS by design — latest.json?flavor=* returns 404 because the manifest handler hits GitHub's /releases/tags/latest (no such tag) instead of /releases/latest. The handler fix is in a separate PR. Refs: #2225, ESPresense/ESPresense#2320. --- .github/workflows/manifest-check.yml | 47 ++++ tools/manifest_check/.gitignore | 2 + tools/manifest_check/README.md | 67 ++++++ tools/manifest_check/check_manifest.py | 238 ++++++++++++++++++++ tools/manifest_check/targets.txt | 26 +++ tools/manifest_check/test_check_manifest.py | 82 +++++++ 6 files changed, 462 insertions(+) create mode 100644 .github/workflows/manifest-check.yml create mode 100644 tools/manifest_check/.gitignore create mode 100644 tools/manifest_check/README.md create mode 100755 tools/manifest_check/check_manifest.py create mode 100644 tools/manifest_check/targets.txt create mode 100644 tools/manifest_check/test_check_manifest.py diff --git a/.github/workflows/manifest-check.yml b/.github/workflows/manifest-check.yml new file mode 100644 index 00000000..f0f9613a --- /dev/null +++ b/.github/workflows/manifest-check.yml @@ -0,0 +1,47 @@ +name: manifest-check + +# Catch issue #2225 ("Failed to download manifest") regressions. Probes the +# public ESP Web Tools manifest endpoints served by functions/releases/[[path]].ts +# and asserts they return well-formed JSON with reachable binaries. + +on: + push: + branches: [main] + paths: + - "tools/manifest_check/**" + - "functions/releases/**" + - ".github/workflows/manifest-check.yml" + pull_request: + paths: + - "tools/manifest_check/**" + - "functions/releases/**" + - ".github/workflows/manifest-check.yml" + schedule: + - cron: "17 * * * *" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + unit-tests: + name: Schema unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 tools/manifest_check/test_check_manifest.py + + probe-manifest: + name: Probe public manifest endpoints + runs-on: ubuntu-latest + needs: unit-tests + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Probe manifest URLs + run: python3 tools/manifest_check/check_manifest.py diff --git a/tools/manifest_check/.gitignore b/tools/manifest_check/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/tools/manifest_check/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/tools/manifest_check/README.md b/tools/manifest_check/README.md new file mode 100644 index 00000000..030c57ef --- /dev/null +++ b/tools/manifest_check/README.md @@ -0,0 +1,67 @@ +# manifest-check + +Headless probe of the public ESP Web Tools manifest endpoints served by this +repo's `functions/releases/[[path]].ts`. Reproduces issue +[#2225](https://github.com/ESPresense/ESPresense/issues/2225) ("Failed to +download manifest"), where `https://espresense.com/releases/{version}.json?flavor={flavor}` +returns 5xx/404 and the web installer surfaces it as a generic "Failed to +download manifest" toast with no diagnostic. + +## Run locally + +```sh +python3 tools/manifest_check/check_manifest.py +``` + +Skip the per-binary HEAD validation (faster, useful when iterating on the +script itself): + +```sh +python3 tools/manifest_check/check_manifest.py --no-check-parts +``` + +Probe a different host (for local dev via `wrangler pages dev`): + +```sh +python3 tools/manifest_check/check_manifest.py --base http://localhost:8788 +``` + +## What the probe checks + +For each `version,flavor` listed in [`targets.txt`](./targets.txt): + +1. `GET /releases/{version}.json?flavor={flavor}` returns HTTP 200 with + `application/json`. +2. The body parses as JSON. +3. Required top-level keys are present (`name`, `version`, `builds`). +4. Each `builds[]` entry has a non-empty `parts[]` array with `path` and + `offset`. +5. Every `parts[].path` HEAD-resolves to 200 / 3xx (catches stale binaries + removed from the CDN). + +Single transient 5xx responses are retried up to 3 times with exponential +backoff before failing the run; this keeps CI from flapping on transient +upstream hiccups while still catching persistent failures like the one +reported on [#2225](https://github.com/ESPresense/ESPresense/issues/2225). + +## Updating the target list + +[`targets.txt`](./targets.txt) is the source of truth for which (version, +flavor) pairs the installer is expected to serve. Edit it when: + +- A new release ships and is added to the installer dropdown. +- A new flavor (board / variant) enters the installer dropdown. +- A release is removed from the public installer (e.g., yanked due to a + bricking bug). + +Lines starting with `#` are comments. Format is `version,flavor` per line. + +## CI + +[`.github/workflows/manifest-check.yml`](../../.github/workflows/manifest-check.yml) +runs the probe on: + +- pull requests that touch this directory, the release handler, or the workflow, +- pushes to `main` that touch the same, +- an hourly cron (`17 * * * *`), +- manual `workflow_dispatch`. diff --git a/tools/manifest_check/check_manifest.py b/tools/manifest_check/check_manifest.py new file mode 100755 index 00000000..f3604b31 --- /dev/null +++ b/tools/manifest_check/check_manifest.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Probe the public ESP Web Tools manifest endpoints for ESPresense releases. + +Reproduces issue #2225 ("Failed to download manifest"), which surfaces when +``https://espresense.com/releases/{version}.json?flavor={flavor}`` returns 5xx +or 404 for a given version/flavor combination. The web installer surfaces that +upstream failure as a generic "Failed to download manifest" toast with no +diagnostic in the UI. + +This script fetches each (version, flavor) pair listed in ``targets.txt``, +asserts the response is HTTP 200 with parseable JSON that conforms to the +ESP Web Tools manifest schema, and HEADs each ``builds[].parts[].path`` to +confirm the binaries are reachable. A small retry budget filters single +transient 500s; persistent failures still fail the run. + +The script is stdlib-only on purpose so it can run unmodified in CI without a +virtualenv setup step. +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +DEFAULT_BASE = "https://espresense.com" +DEFAULT_TARGETS = Path(__file__).with_name("targets.txt") +USER_AGENT = "ESPresense-manifest-check/1 (+https://github.com/ESPresense/ESPresense.com)" + +REQUIRED_TOP_KEYS = ("name", "version", "builds") +REQUIRED_PART_KEYS = ("path", "offset") + + +@dataclass +class Target: + version: str + flavor: str + + def manifest_url(self, base: str) -> str: + query = urllib.parse.urlencode({"flavor": self.flavor}) + return f"{base}/releases/{self.version}.json?{query}" + + +@dataclass +class Failure: + target: Target + stage: str + detail: str + + +@dataclass +class Report: + failures: list[Failure] = field(default_factory=list) + checked: int = 0 + + def fail(self, target: Target, stage: str, detail: str) -> None: + self.failures.append(Failure(target, stage, detail)) + + +def parse_targets(path: Path) -> list[Target]: + targets: list[Target] = [] + for raw in path.read_text().splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + if "," not in line: + raise SystemExit(f"{path}: malformed line (expected 'version,flavor'): {raw!r}") + version, flavor = (part.strip() for part in line.split(",", 1)) + if not version or not flavor: + raise SystemExit(f"{path}: empty version or flavor: {raw!r}") + targets.append(Target(version=version, flavor=flavor)) + if not targets: + raise SystemExit(f"{path}: no targets") + return targets + + +def http_request(url: str, *, method: str, timeout: float, retries: int, backoff: float) -> tuple[int, bytes, str]: + last_exc: Exception | None = None + for attempt in range(retries + 1): + req = urllib.request.Request(url, method=method, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read() if method == "GET" else b"" + return resp.status, body, resp.headers.get("Content-Type", "") + except urllib.error.HTTPError as exc: + if (exc.code < 500 and exc.code != 429) or attempt == retries: + body = exc.read() if method == "GET" else b"" + return exc.code, body, exc.headers.get("Content-Type", "") if exc.headers else "" + last_exc = exc + except (urllib.error.URLError, TimeoutError) as exc: + last_exc = exc + if attempt == retries: + raise + time.sleep(backoff * (2 ** attempt)) + if last_exc: + raise last_exc + raise RuntimeError("unreachable") + + +def validate_schema(payload: dict) -> str | None: + for key in REQUIRED_TOP_KEYS: + if key not in payload: + return f"missing top-level key: {key}" + builds = payload.get("builds") + if not isinstance(builds, list) or not builds: + return "builds[] must be a non-empty list" + for idx, build in enumerate(builds): + if not isinstance(build, dict): + return f"builds[{idx}] is not an object" + if "chipFamily" not in build: + return f"builds[{idx}] missing chipFamily" + parts = build.get("parts") + if not isinstance(parts, list) or not parts: + return f"builds[{idx}].parts must be a non-empty list" + for pidx, part in enumerate(parts): + if not isinstance(part, dict): + return f"builds[{idx}].parts[{pidx}] is not an object" + for key in REQUIRED_PART_KEYS: + if key not in part: + return f"builds[{idx}].parts[{pidx}] missing {key}" + return None + + +def check_target( + target: Target, base: str, *, timeout: float, retries: int, backoff: float, check_parts: bool, report: Report +) -> None: + report.checked += 1 + url = target.manifest_url(base) + try: + status, body, content_type = http_request( + url, method="GET", timeout=timeout, retries=retries, backoff=backoff + ) + except Exception as exc: # noqa: BLE001 -- want the message verbatim + report.fail(target, "fetch", f"{url}: {exc}") + return + + if status != 200: + snippet = body[:200].decode("utf-8", errors="replace").strip() + report.fail(target, "status", f"{url}: HTTP {status} (body: {snippet!r})") + return + + if "json" not in content_type.lower(): + report.fail(target, "content-type", f"{url}: got {content_type!r}, want application/json") + return + + try: + payload = json.loads(body) + except json.JSONDecodeError as exc: + snippet = body[:200].decode("utf-8", errors="replace") + report.fail(target, "parse", f"{url}: {exc.msg} at offset {exc.pos} (body starts: {snippet!r})") + return + + schema_error = validate_schema(payload) + if schema_error: + report.fail(target, "schema", f"{url}: {schema_error}") + return + + if not check_parts: + return + + seen_paths: set[str] = set() + for build in payload["builds"]: + for part in build["parts"]: + path = part["path"] + if path in seen_paths: + continue + seen_paths.add(path) + if path.startswith(("http://", "https://")): + part_url = path + else: + part_url = urllib.parse.urljoin(url, path) + try: + pstatus, _, _ = http_request( + part_url, method="HEAD", timeout=timeout, retries=retries, backoff=backoff + ) + except Exception as exc: # noqa: BLE001 + report.fail(target, "part-fetch", f"{part_url}: {exc}") + continue + if pstatus not in (200, 301, 302, 307, 308): + report.fail(target, "part-status", f"{part_url}: HTTP {pstatus}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n", 1)[0]) + parser.add_argument( + "--base", default=DEFAULT_BASE, help="Manifest host (default: %(default)s)" + ) + parser.add_argument( + "--targets", + type=Path, + default=DEFAULT_TARGETS, + help="Path to a 'version,flavor' targets file (default: %(default)s)", + ) + parser.add_argument( + "--timeout", type=float, default=15.0, help="Per-request timeout seconds (default: %(default)s)" + ) + parser.add_argument( + "--retries", type=int, default=3, help="Retry count for 5xx and connection errors (default: %(default)s)" + ) + parser.add_argument( + "--backoff", type=float, default=2.0, help="Initial backoff seconds, exponential (default: %(default)s)" + ) + parser.add_argument( + "--no-check-parts", action="store_true", help="Skip the per-binary HEAD validation" + ) + args = parser.parse_args(argv) + + targets = parse_targets(args.targets) + report = Report() + for target in targets: + check_target( + target, + args.base.rstrip("/"), + timeout=args.timeout, + retries=args.retries, + backoff=args.backoff, + check_parts=not args.no_check_parts, + report=report, + ) + + print(f"Checked {report.checked} target(s).") + if not report.failures: + print("All manifests OK.") + return 0 + + print(f"FAILED: {len(report.failures)} problem(s):", file=sys.stderr) + for f in report.failures: + print(f" [{f.stage}] {f.target.version}/{f.target.flavor}: {f.detail}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/manifest_check/targets.txt b/tools/manifest_check/targets.txt new file mode 100644 index 00000000..3b607f57 --- /dev/null +++ b/tools/manifest_check/targets.txt @@ -0,0 +1,26 @@ +# ESPresense web-installer manifest endpoints to probe. +# Format: version,flavor (one pair per line; '#' starts a comment) +# +# Add a row when a release ships or when a new flavor enters the installer +# dropdown. Drop a row when a release is removed from the public installer. +# +# Reference: issue #2225 ("Failed to download manifest"). Web installer fetches +# https://espresense.com/releases/{version}.json?flavor={flavor} per selection. + +# 'latest' is whatever the installer's default points at; it must always work. +latest,esp32 +latest,esp32c3 +latest,esp32s3 +latest,esp32c6 +latest,m5atom +latest,m5stickc + +# v4.0.6 is the version flagged in #2225 with persistent intermittent 500s. +v4.0.6,esp32 +v4.0.6,esp32c3 +v4.0.6,esp32s3 +v4.0.6,m5atom + +# v4.0.5 is the last-known-working release reported on #2225; keep as a sentinel. +v4.0.5,esp32 +v4.0.5,m5atom diff --git a/tools/manifest_check/test_check_manifest.py b/tools/manifest_check/test_check_manifest.py new file mode 100644 index 00000000..2e4bfa3b --- /dev/null +++ b/tools/manifest_check/test_check_manifest.py @@ -0,0 +1,82 @@ +"""Unit tests for the manifest-check schema validator. + +Network-touching paths are exercised in CI by running the script against the +real manifest endpoint; this file only covers the offline parser/schema logic +so it can run as a fast PR-time check without external dependencies. +""" +from __future__ import annotations + +import unittest +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent)) +from check_manifest import parse_targets, validate_schema # noqa: E402 + + +class ValidateSchemaTests(unittest.TestCase): + def _good(self) -> dict: + return { + "name": "ESPresense vX (esp32)", + "version": "vX", + "new_install_prompt_erase": True, + "builds": [ + { + "chipFamily": "ESP32", + "parts": [ + {"path": "/static/esp32/bootloader.bin", "offset": 4096}, + {"path": "download/vX/esp32.bin", "offset": 65536}, + ], + } + ], + } + + def test_valid(self) -> None: + self.assertIsNone(validate_schema(self._good())) + + def test_missing_top_key(self) -> None: + for key in ("name", "version", "builds"): + payload = self._good() + del payload[key] + with self.subTest(key=key): + err = validate_schema(payload) + self.assertIsNotNone(err) + self.assertIn(key, err) + + def test_empty_builds(self) -> None: + payload = self._good() + payload["builds"] = [] + self.assertIn("non-empty", validate_schema(payload) or "") + + def test_part_missing_offset(self) -> None: + payload = self._good() + del payload["builds"][0]["parts"][0]["offset"] + err = validate_schema(payload) + self.assertIsNotNone(err) + self.assertIn("offset", err) + + def test_build_missing_parts(self) -> None: + payload = self._good() + del payload["builds"][0]["parts"] + err = validate_schema(payload) + self.assertIsNotNone(err) + self.assertIn("parts", err) + + +class TargetsTests(unittest.TestCase): + def test_parse(self) -> None: + from tempfile import NamedTemporaryFile + with NamedTemporaryFile("w", suffix=".txt", delete=False) as fh: + fh.write("# header\nlatest,esp32\n v4.0.6, m5atom # legacy\n\n") + path = Path(fh.name) + try: + targets = parse_targets(path) + self.assertEqual(len(targets), 2) + self.assertEqual((targets[0].version, targets[0].flavor), ("latest", "esp32")) + self.assertEqual((targets[1].version, targets[1].flavor), ("v4.0.6", "m5atom")) + finally: + path.unlink() + + +if __name__ == "__main__": + unittest.main()