From 15008ac45fcc76f9311b0e931efc0e39b600f0c9 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere <9342524+samlaf@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:19:16 +0800 Subject: [PATCH] feat(deploy_tee): genesis-ceremony polls for cohort readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ceremony's two probes — each node's reth serving block 0 and summit's getPublicKeys — only answer after a node finishes its root_key → LUKS open → summit keygen boot tail, and both previously read once and aborted, so the operator had to hand-time the ceremony against the slowest node. Both probes now poll until every node answers (5s interval, 15 min ceiling, pending-list progress every 30s), making the ceremony itself the cohort barrier: it can run straight after `configure` in one sitting. One deliberate asymmetry: an unreachable node is retried (that's the normal boot tail), but a node that *answers* with the wrong genesis hash fails immediately with the full per-node ✓/✗ listing — waiting can't fix a node booted from a stale image or different genesis. README, from the first 4-node founding: show cohort configure (--genesis/--join) as the founding path, document the ceremony's readiness wait, note tdx-init's one-shot listener ("listener never came up" on an already-configured node means reboot it first), and record the stack-per-node provisioning rationale. --- deploy_tee/README.md | 51 +++++---- deploy_tee/genesis.py | 187 ++++++++++++++++++++++--------- deploy_tee/tests/test_genesis.py | 94 +++++++++++++++- 3 files changed, 259 insertions(+), 73 deletions(-) diff --git a/deploy_tee/README.md b/deploy_tee/README.md index 5dc88f9..c9eee9d 100644 --- a/deploy_tee/README.md +++ b/deploy_tee/README.md @@ -30,8 +30,8 @@ for its [InitConfig](https://github.com/SeismicSystems/enclave/blob/seismic/crat VHD upload to the Azure image registry is a seismic-images concern (`make push-azure-*`), not this tool. - **For `seismic-tee-bootstrap genesis-ceremony`:** the `genesis` binary (built from - summit) on PATH, plus a summit genesis template passed via - `--summit-template`. + summit) on PATH. (The summit genesis template defaults to the + artifact-set copy beside `--manifest`.) - **For `seismic-tee-bootstrap up`** (cohort provisioning): the `pulumi` CLI on PATH. The passphrase secrets provider needs `PULUMI_CONFIG_PASSPHRASE` — set it, or let `up`/`down` prompt you on @@ -77,6 +77,13 @@ small JSON (`public_ip`/`fqdn`) describing one provisioned node — see (just `{public_ip, fqdn}`); a BYO-infra operator can hand-write it (or use `pulumi stack output --json` — only those two keys are read). +Cohort provisioning is one Pulumi stack **per node** (not one cohort +stack), driven as a *local* Automation-API program: any node can be +recycled or torn down without touching the others, and every stack stays +manageable with the plain `pulumi` CLI (`pulumi stack output` / `up` / +`destroy --stack …`) — the orchestrator and the CLI are interchangeable +front-ends over the same state. + ### Where this is heading: a public operator repo Longer-term these two CLIs are intended to live in **two repos**, split on @@ -162,19 +169,21 @@ uv run seismic-tee-bootstrap manifest assemble deploy_tee/networks/devnet-3 uv run seismic-tee-bootstrap up --count 2 --network deploy_tee/networks/devnet-3 # → deploy_tee/networks/devnet-3/nodes/dev-bootstrap-node-{1,2}.json -# 3. Configure each node against the shared manifest and POST. Node 1 founds -# the network (genesis, via the bootstrap CLI); node 2 joins (operator CLI), -# fetching root_key from node 1. +# 3. Configure the whole cohort in one command: exactly one genesis node +# (mints root_key) plus N joiners pointed at it (each fetches root_key +# from node 1 over the attested handshake); every first-boot disk wipe +# is watched in parallel. (A node joining the live network later uses +# `seismic-tee configure` instead — see "Joining an existing network".) uv run seismic-tee-bootstrap configure \ --genesis deploy_tee/networks/devnet-3/nodes/dev-bootstrap-node-1.json \ - --manifest deploy_tee/networks/devnet-3/network-manifest.json -uv run seismic-tee configure \ - --node deploy_tee/networks/devnet-3/nodes/dev-bootstrap-node-2.json \ - --peer deploy_tee/networks/devnet-3/nodes/dev-bootstrap-node-1.json \ + --join deploy_tee/networks/devnet-3/nodes/dev-bootstrap-node-2.json \ --manifest deploy_tee/networks/devnet-3/network-manifest.json # 4. Run the genesis ceremony once: builds genesis.toml from the cohort # (needs the `genesis` binary on PATH) and fans it out to every summit. +# Waits for cohort readiness itself — polls until each node's reth +# serves the manifest's genesis hash as block 0 and its summit pubkeys +# are readable — so it can run straight after step 3. # One-shot: send_genesis only exists in the pre-genesis node API, so this # can never be re-run against a live network. --summit-template defaults # to the artifact-set copy beside --manifest (verified against the @@ -227,7 +236,10 @@ back as 4xx and the deploy script raises. `configure` POSTs this body verbatim to the VM's `http://:8080/`, retrying for 180s while the listener comes up (LUKS format + mkfs + TPM enroll takes ~20–40s on first -boot). On success tdx-init writes: +boot). tdx-init is a one-shot listener: once a boot's POST is +consumed, `:8080` stays down until the next reboot — "listener +never came up" against an already-configured node means *reboot it +first*, not that the node is broken. On success tdx-init writes: | File | Content | Consumer | | ----------------------------------------- | --------------------------------------------------------------- | --------------------------- | @@ -242,13 +254,15 @@ only gates against a second POST within the same boot. ### Operator footguns to be aware of -- **Running `seismic-tee-bootstrap configure` against more than one VM - silently splits the network.** Each genesis enclave generates a fresh - `OsRng` `root_key` locally; downstream nodes that fetch from one cohort - can't decrypt state from the other. There is no in-band check yet — the - planned parallel cohort founder assigns exactly one genesis, making this - unrepresentable. (The operator `seismic-tee configure` can't trigger it: it - only joins.) +- **Two root_key-genesis nodes silently split the network.** Each genesis + enclave generates a fresh `OsRng` `root_key` locally; downstream nodes + that fetch from one cohort can't decrypt state from the other. + `seismic-tee-bootstrap configure` makes this unrepresentable within one + invocation (exactly one `--genesis`), but nothing yet stops a second + *invocation* against a fresh VM — a durable guard (genesis mode refusing + to start against an already-formatted volume or a live peer) is planned + upstream in tdx-init. (The operator `seismic-tee configure` can't + trigger it: it only joins.) - **The `:8080` listener is unauthenticated, first-POST-wins.** An attacker who reaches port 8080 ahead of the operator can write a malicious `peers` list, redirecting `root_key` fetch to @@ -268,8 +282,7 @@ serves that RA-TLS endpoint, and cvm-reverse-proxy's pre-handshake aTLS is a different protocol than the `flashbots/attested-tls` the enclave standardizes on. Verification will be reimplemented against attested-tls / `seismic-attestation` (the enclave already exposes -`getAttestationEvidence` plus a deploy-preflight binding). Tracked in -`plans/tee`. +`getAttestationEvidence` plus a deploy-preflight binding). When it returns, the design is **bring-your-own measurements**: the operator supplies the image's expected measurements (published by diff --git a/deploy_tee/genesis.py b/deploy_tee/genesis.py index fbdcda1..faef294 100644 --- a/deploy_tee/genesis.py +++ b/deploy_tee/genesis.py @@ -13,6 +13,11 @@ booted from a stale image or wrong genesis fails the ceremony loudly instead of parking summit in SYNCING forever. +The ceremony doubles as the cohort barrier: its two probes (reth block 0, +summit `getPublicKeys`) answer only after a node finishes root_key → LUKS +open → summit keygen, and both are polled until every node responds — so +it can run straight after `configure` without hand-timing the boot tail. + Each node is located by a descriptor file (see deploy_tee/descriptor.py), so this never calls Pulumi. """ @@ -23,18 +28,27 @@ import shutil import subprocess import tempfile +import time from pathlib import Path import requests from deploy_tee import manifest as manifest_mod from deploy_tee.descriptor import load_descriptor, require -from deploy_tee.utils.summit_client import SummitClient +from deploy_tee.utils.summit_client import PublicKeys, SummitClient # Summit's consensus (BLS) port. Each validator entry in the generated # genesis.toml pins ":". CONSENSUS_PORT = 18551 +# Cohort-readiness polling (the barrier). `configure` already watches the +# 1h+ first-boot disk wipes, so by ceremony time the residual boot tail is +# minutes; the timeout turns a wedged node into a loud per-node report +# instead of hanging the ceremony indefinitely. +POLL_INTERVAL_SECONDS = 5 +READY_TIMEOUT_SECONDS = 15 * 60 +WAIT_LOG_INTERVAL_SECONDS = 30 + # anvil/sanvil dev accounts: the first 10 addresses from the well-known # "test test test ... junk" mnemonic. Their private keys are public, so # these are devnet-only — never use them as withdrawal_credentials on a @@ -150,6 +164,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def _get_pubkeys( descriptors: list[Path], + *, + timeout: float = READY_TIMEOUT_SECONDS, + interval: float = POLL_INTERVAL_SECONDS, ) -> tuple[list[dict[str, str]], list[tuple[Path, SummitClient]]]: """Gather each cohort node's summit pubkeys from its descriptor file. @@ -158,30 +175,62 @@ def _get_pubkeys( infra tool produced the descriptor — this script never calls Pulumi. Returns the validator entries and the per-node summit clients (reused for the genesis.toml fanout). + + Each node is polled until its pubkeys are readable: getPublicKeys + answers only once summit is up with generated keys (after root_key → + LUKS open → keygen), so an unreadable node is normally just still + booting. A node still unreadable at `timeout` aborts the ceremony with + a per-node report. """ - validators = [] node_clients: list[tuple[Path, SummitClient]] = [] - for i, path in enumerate(descriptors): + ip_addresses: dict[Path, str] = {} + for path in descriptors: descriptor = load_descriptor(path) fqdn = require(descriptor, "fqdn", path) - ip_address = require(descriptor, "public_ip", path) - client = SummitClient(f"https://{fqdn}/summit") - node_clients.append((path, client)) - try: - pubkeys = client.get_public_keys() - validators.append( - { - "node_public_key": pubkeys.node, - "consensus_public_key": pubkeys.consensus, - "ip_address": f"{ip_address}:{CONSENSUS_PORT}", - "withdrawal_credentials": _ANVIL_ADDRESSES[ - i % len(_ANVIL_ADDRESSES) - ], - } + ip_addresses[path] = require(descriptor, "public_ip", path) + node_clients.append((path, SummitClient(f"https://{fqdn}/summit"))) + + pubkeys: dict[Path, PublicKeys] = {} + last_error: dict[Path, str] = {} + deadline = time.monotonic() + timeout + next_log = 0.0 + while True: + for path, client in node_clients: + if path in pubkeys: + continue + try: + pubkeys[path] = client.get_public_keys() + print(f" ✓ {path.stem}: summit pubkeys readable") + except Exception as e: + last_error[path] = str(e) + pending = [path for path in descriptors if path not in pubkeys] + if not pending: + break + now = time.monotonic() + if now >= deadline: + listing = "\n".join(f" ✗ {path}: {last_error[path]}" for path in pending) + raise SystemExit( + f"{len(pending)} node(s) still without readable summit pubkeys " + f"after {int(timeout)}s (stuck before keygen?); aborting the " + f"ceremony:\n{listing}" ) - except Exception as e: - print(f"Error: {e}") - raise e + if now >= next_log: + print( + "waiting for summit pubkeys: " + + ", ".join(path.stem for path in pending) + ) + next_log = now + WAIT_LOG_INTERVAL_SECONDS + time.sleep(interval) + + validators = [ + { + "node_public_key": pubkeys[path].node, + "consensus_public_key": pubkeys[path].consensus, + "ip_address": f"{ip_addresses[path]}:{CONSENSUS_PORT}", + "withdrawal_credentials": _ANVIL_ADDRESSES[i % len(_ANVIL_ADDRESSES)], + } + for i, path in enumerate(descriptors) + ] return validators, node_clients @@ -208,7 +257,13 @@ def _verify_template_commitment(template_path: Path, manifest: dict) -> None: ) -def _assert_cohort_genesis_hash(descriptors: list[Path], expected: str) -> None: +def _assert_cohort_genesis_hash( + descriptors: list[Path], + expected: str, + *, + timeout: float = READY_TIMEOUT_SECONDS, + interval: float = POLL_INTERVAL_SECONDS, +) -> None: """Assert every cohort node's reth serves `expected` as block 0. summit's genesis `eth_genesis_hash` must equal reth's real genesis hash: @@ -219,45 +274,75 @@ def _assert_cohort_genesis_hash(descriptors: list[Path], expected: str) -> None: node booted from a different image/genesis before the ceremony pins the validator set — and doubles as a reth-readiness probe before send_genesis. + + A node that doesn't answer is polled until `timeout` — reth comes up + only after root_key → LUKS open, so early unreachability is the normal + boot tail. A *wrong* answer fails immediately: waiting can't fix a node + booted from a stale image or different genesis. """ - # Query every node before judging, and report the full cohort on failure — - # a partial listing is ambiguous with "stopped at the first bad node", and - # which nodes match is exactly the diagnostic (one stale node vs. a - # manifest that matches nobody). - observed: dict[Path, str] = {} # block-0 hash, or an error description + urls: dict[Path, str] = {} for path in descriptors: descriptor = load_descriptor(path) fqdn = require(descriptor, "fqdn", path) - url = f"https://{fqdn}/rpc" # nginx proxies /rpc -> reth :8545 - try: - response = requests.post( - url, - json={ - "jsonrpc": "2.0", - "id": 1, - "method": "eth_getBlockByNumber", - "params": ["0x0", False], - }, - timeout=30, - ) - response.raise_for_status() - data = response.json() - if data.get("result") is None: - raise RuntimeError( - f"eth_getBlockByNumber returned {data.get('error') or data}" + urls[path] = f"https://{fqdn}/rpc" # nginx proxies /rpc -> reth :8545 + + observed: dict[Path, str] = {} # block-0 hash, once a node has answered + last_error: dict[Path, str] = {} + deadline = time.monotonic() + timeout + next_log = 0.0 + while True: + for path in descriptors: + if path in observed: + continue + try: + response = requests.post( + urls[path], + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": ["0x0", False], + }, + timeout=30, ) - observed[path] = data["result"]["hash"] - except Exception as e: - observed[path] = f"unreachable via {url}: {e}" - if any(h.lower() != expected.lower() for h in observed.values()): + response.raise_for_status() + data = response.json() + if data.get("result") is None: + raise RuntimeError( + f"eth_getBlockByNumber returned {data.get('error') or data}" + ) + observed[path] = data["result"]["hash"] + except Exception as e: + last_error[path] = f"unreachable via {urls[path]}: {e}" + pending = [path for path in descriptors if path not in observed] + mismatch = any(h.lower() != expected.lower() for h in observed.values()) + if mismatch or not pending: + break + now = time.monotonic() + if now >= deadline: + break + if now >= next_log: + print( + "waiting for reth block 0: " + ", ".join(path.stem for path in pending) + ) + next_log = now + WAIT_LOG_INTERVAL_SECONDS + time.sleep(interval) + + if mismatch or pending: + # Report the full cohort, not just the first bad node — a partial + # listing is ambiguous with "stopped at the first bad node", and + # which nodes match is exactly the diagnostic (one stale node vs. a + # manifest that matches nobody). listing = "\n".join( f" {'✓' if h.lower() == expected.lower() else '✗'} {path}: {h}" - for path, h in observed.items() + for path, h in ( + (path, observed.get(path) or last_error[path]) for path in descriptors + ) ) raise SystemExit( - "Cohort disagrees with the declared eth_genesis_hash (stale image " - "or wrong reth genesis?); refusing to build genesis.toml:\n" - f" declared: {expected}\n{listing}" + "Cohort disagrees with the declared eth_genesis_hash (stale image, " + "wrong reth genesis, or a node that never became ceremony-ready); " + f"refusing to build genesis.toml:\n declared: {expected}\n{listing}" ) diff --git a/deploy_tee/tests/test_genesis.py b/deploy_tee/tests/test_genesis.py index 9ec2fa5..952a0b9 100644 --- a/deploy_tee/tests/test_genesis.py +++ b/deploy_tee/tests/test_genesis.py @@ -12,6 +12,7 @@ from unittest import mock from deploy_tee import genesis +from deploy_tee.utils.summit_client import PublicKeys class ParseArgsTests(unittest.TestCase): @@ -181,7 +182,7 @@ def test_mismatching_node_exits_listing_whole_cohort(self): self.assertIn(self.OTHER_HASH, msg) def test_unreachable_node_reported_without_hiding_others(self): - # First node down; second still queried and reported. + # First node down past the deadline; second still queried and reported. nodes = [ self._descriptor("node-1", "a.example"), self._descriptor("node-2", "b.example"), @@ -189,7 +190,7 @@ def test_unreachable_node_reported_without_hiding_others(self): responses = [OSError("boom"), self._block_resp(self.HASH)] with mock.patch.object(genesis.requests, "post", side_effect=responses): with self.assertRaises(SystemExit) as ctx: - genesis._assert_cohort_genesis_hash(nodes, self.HASH) + genesis._assert_cohort_genesis_hash(nodes, self.HASH, timeout=0) msg = str(ctx.exception) self.assertIn("https://a.example/rpc", msg) self.assertIn(f"✓ {nodes[1]}", msg) @@ -199,9 +200,96 @@ def test_rpc_error_response_exits(self): resp = self._resp({"jsonrpc": "2.0", "id": 1, "error": {"message": "boom"}}) with mock.patch.object(genesis.requests, "post", return_value=resp): with self.assertRaises(SystemExit) as ctx: - genesis._assert_cohort_genesis_hash(nodes, self.HASH) + genesis._assert_cohort_genesis_hash(nodes, self.HASH, timeout=0) self.assertIn("boom", str(ctx.exception)) + def test_unreachable_node_polled_until_it_answers(self): + # The ceremony is the cohort barrier: reth comes up only after the + # root_key → LUKS boot tail, so a node that doesn't answer yet is + # retried rather than failing the ceremony. + nodes = [self._descriptor("node-1", "a.example")] + responses = [OSError("still booting"), self._block_resp(self.HASH)] + with mock.patch.object(genesis.requests, "post", side_effect=responses) as post: + genesis._assert_cohort_genesis_hash( + nodes, self.HASH, timeout=30, interval=0 + ) + self.assertEqual(post.call_count, 2) + + def test_mismatch_fails_fast_without_waiting_for_stragglers(self): + # A wrong answer can't heal by waiting — fail immediately even while + # another node is still unreachable, instead of burning the timeout. + nodes = [ + self._descriptor("node-1", "a.example"), + self._descriptor("node-2", "b.example"), + ] + responses = [self._block_resp(self.OTHER_HASH), OSError("still booting")] + no_sleep = mock.patch.object( + genesis.time, "sleep", side_effect=AssertionError("must not wait") + ) + with mock.patch.object(genesis.requests, "post", side_effect=responses): + with no_sleep, self.assertRaises(SystemExit) as ctx: + genesis._assert_cohort_genesis_hash(nodes, self.HASH) + msg = str(ctx.exception) + self.assertIn(self.OTHER_HASH, msg) + self.assertIn("still booting", msg) + + +class GetPubkeysTests(unittest.TestCase): + """The pubkey gather is the other half of the cohort barrier: summit + answers getPublicKeys only after root_key → LUKS → keygen, so unreadable + nodes are polled until ready (bounded), not failed on the first read.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + + def _descriptor(self, name: str, fqdn: str, ip: str = "203.0.113.1") -> Path: + path = Path(self._tmp.name) / f"{name}.json" + path.write_text(json.dumps({"public_ip": ip, "fqdn": fqdn})) + return path + + def test_waits_until_pubkeys_readable(self): + node = self._descriptor("node-1", "a.example") + client = mock.Mock() + client.get_public_keys.side_effect = [ + OSError("502 Bad Gateway"), + PublicKeys(node="0xnode", consensus="0xbls"), + ] + with mock.patch.object(genesis, "SummitClient", return_value=client): + validators, clients = genesis._get_pubkeys([node], timeout=30, interval=0) + self.assertEqual(client.get_public_keys.call_count, 2) + self.assertEqual( + validators, + [ + { + "node_public_key": "0xnode", + "consensus_public_key": "0xbls", + "ip_address": f"203.0.113.1:{genesis.CONSENSUS_PORT}", + "withdrawal_credentials": genesis._ANVIL_ADDRESSES[0], + } + ], + ) + self.assertEqual(clients, [(node, client)]) + + def test_timeout_lists_only_the_stuck_nodes(self): + ready = self._descriptor("node-1", "a.example") + stuck = self._descriptor("node-2", "b.example") + ready_client = mock.Mock() + ready_client.get_public_keys.return_value = PublicKeys( + node="0xnode", consensus="0xbls" + ) + stuck_client = mock.Mock() + stuck_client.get_public_keys.side_effect = OSError("boom") + with mock.patch.object( + genesis, "SummitClient", side_effect=[ready_client, stuck_client] + ): + with self.assertRaises(SystemExit) as ctx: + genesis._get_pubkeys([ready, stuck], timeout=0) + msg = str(ctx.exception) + self.assertIn(str(stuck), msg) + self.assertIn("boom", msg) + self.assertNotIn(str(ready), msg) + if __name__ == "__main__": unittest.main()