RaoFoundation · UnArbosFive · Jul 17, 2026
diff --git a/.github/scripts/classify-runtime-changes.sh b/.github/scripts/classify-runtime-changes.sh
@@ -40,7 +40,7 @@ while IFS= read -r path; do
   esac
 
   case "$path" in
-    sdk/python/*|sdk/bittensor-core/*|sdk/bittensor-core-py/*|sdk/bittensor-core-wasm/*|Cargo.lock|.github/workflows/runtime-checks.yml)
+    sdk/python/*|sdk/bittensor-core/*|sdk/bittensor-core-py/*|sdk/bittensor-core-wasm/*|Cargo.lock|.github/workflows/runtime-checks.yml|.github/scripts/prepare-sdk-dist.py|.github/scripts/test_prepare_sdk_dist.py)
       python_sdk=true
       ;;
   esac

diff --git a/.github/scripts/prepare-sdk-dist.py b/.github/scripts/prepare-sdk-dist.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Stamp the SDK's committed development version for a stable PyPI build."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def main() -> None:
+    root = Path("sdk/python")
+    manifest = root / "pyproject.toml"
+    text = manifest.read_text()
+    match = re.search(r'^version = "([0-9]+\.[0-9]+\.[0-9]+)\.dev0"$', text, flags=re.M)
+    if match is None:
+        raise SystemExit("SDK manifest must declare version X.Y.Z.dev0")
+    version = match.group(1)
+    text = text[: match.start()] + f'version = "{version}"' + text[match.end() :]
+
+    # These are monorepo-only development inputs. They must not appear in an
+    # extracted sdist, where the sibling path and development lock do not exist.
+    text, count = re.subn(r"\n\[tool\.uv\.sources\]\n.*?(?=\n\[|\Z)", "\n", text, flags=re.S)
+    if count != 1:
+        raise SystemExit("tool.uv.sources table not found")
+    manifest.write_text(text)
+    (root / "uv.lock").unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/scripts/test-runtime-change-filter.sh b/.github/scripts/test-runtime-change-filter.sh
@@ -33,6 +33,8 @@ assert_classification .github/workflows/refresh-mainnet-snapshot.yml "$runtime_a
 assert_classification .github/workflows/runtime-checks.yml "$runtime_and_snapshot"
 assert_classification website/apps/bittensor-website/scripts/generate-metadata.mjs "$runtime_and_docs"
 assert_classification sdk/bittensor-core/src/lib.rs "$python_only"
+assert_classification .github/scripts/prepare-sdk-dist.py "$python_only"
+assert_classification .github/scripts/test_prepare_sdk_dist.py "$python_only"
 assert_classification Cargo.lock "$python_only"
 assert_classification rust-toolchain.toml "$runtime_and_sdk"
 assert_classification $'README.md\nsdk/python/example.py' "$docs_and_python"

diff --git a/.github/scripts/test_prepare_sdk_dist.py b/.github/scripts/test_prepare_sdk_dist.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+class PrepareSdkDistributionTests(unittest.TestCase):
+    def test_stamps_stable_version_and_removes_monorepo_inputs(self) -> None:
+        repository = Path(__file__).resolve().parents[2]
+        script = Path(__file__).with_name("prepare-sdk-dist.py")
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            package = root / "sdk/python"
+            package.mkdir(parents=True)
+            source_manifest = repository / "sdk/python/pyproject.toml"
+            source_text = source_manifest.read_text()
+            version = re.search(
+                r'^version = "([0-9]+\.[0-9]+\.[0-9]+)\.dev0"$', source_text, flags=re.M
+            )
+            self.assertIsNotNone(version)
+            shutil.copy(source_manifest, package / "pyproject.toml")
+            shutil.copy(repository / "sdk/python/uv.lock", package / "uv.lock")
+            subprocess.run(
+                [sys.executable, str(script)],
+                cwd=root,
+                check=True,
+            )
+
+            manifest = (package / "pyproject.toml").read_text()
+            self.assertIn(f'version = "{version.group(1)}"', manifest)
+            self.assertNotIn("[tool.uv.sources]", manifest)
+            self.assertFalse((package / "uv.lock").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/.github/workflows/runtime-checks.yml b/.github/workflows/runtime-checks.yml
@@ -439,6 +439,7 @@ jobs:
       - name: SDK offline checks (same as just check)
         working-directory: sdk/python
         run: |
+          python3 ../../.github/scripts/test_prepare_sdk_dist.py
           uv sync --locked --all-extras --dev
           uv run ruff check .
           uv run ruff format --check .

diff --git a/.github/workflows/watch-mainnet-release.yml b/.github/workflows/watch-mainnet-release.yml
@@ -2,8 +2,9 @@ name: Watch Mainnet Release
 
 # The mainnet upgrade is executed by the triumvirate signing the multisig
 # proposal submitted by the release train (release-train.yml). This watcher
-# polls the chain; once the on-chain spec_version matches main and no
-# release exists for it yet, it cuts the release:
+# polls the chain; once an upgrade executes, it cuts the release. Stable Python
+# publication has its own completion marker so a later run can retry a failed
+# or partial PyPI upload even when the GitHub release already exists:
 #   1. GitHub release v<spec_version>, with the srtool wasm + digest and
 #      multisig call data from the release train attached as assets
 #   2. Docker images via explicit dispatch of docker.yml and
@@ -40,72 +41,107 @@ jobs:
       contents: read
       actions: read # list/download the release-train artifact
     outputs:
-      ready: ${{ steps.compare.outputs.ready }}
+      release_needed: ${{ steps.compare.outputs.release_needed }}
+      python_needed: ${{ steps.compare.outputs.python_needed }}
       spec_version: ${{ steps.compare.outputs.spec_version }}
       sha: ${{ steps.compare.outputs.sha }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Compare spec versions and existing releases
+      - name: Compare mainnet with release state
         id: compare
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
+          set -euo pipefail
+
           local_spec=$(grep -Eo 'spec_version: *[0-9]+' runtime/src/lib.rs | head -n 1 | grep -Eo '[0-9]+')
           : ${local_spec:?could not parse spec_version from runtime/src/lib.rs}
 
           chain_spec=$(curl -sf -H "Content-Type: application/json" \
             -d '{"id":1,"jsonrpc":"2.0","method":"state_getRuntimeVersion","params":[]}' \
             "$MAINNET_HTTP" | jq -r '.result.specVersion')
           : ${chain_spec:?could not fetch chain spec_version}
+          [[ "$local_spec" =~ ^[0-9]+$ && "$chain_spec" =~ ^[0-9]+$ ]] \
+            || { echo "spec_version values must be integers"; exit 1; }
 
           echo "main spec_version:     $local_spec"
           echo "on-chain spec_version: $chain_spec"
 
-          echo "spec_version=$local_spec" >> $GITHUB_OUTPUT
-
-          if [[ "$chain_spec" != "$local_spec" ]]; then
-            echo "Upgrade not executed yet (or main is ahead); nothing to do."
-            echo "ready=false" >> $GITHUB_OUTPUT
+          if [ "$chain_spec" -gt "$local_spec" ]; then
+            echo "Main does not contain the on-chain runtime yet; refusing to publish."
+            echo "release_needed=false" >> "$GITHUB_OUTPUT"
+            echo "python_needed=false" >> "$GITHUB_OUTPUT"
             exit 0
           fi
+          echo "spec_version=$chain_spec" >> "$GITHUB_OUTPUT"
 
           # Match both this workflow's tags (v424) and the legacy scheme
-          # inherited from upstream (v3.4.9-424) so an already-released
-          # runtime is never released twice. Pre-releases don't count: the
-          # release train publishes the proposal as a pre-release v<spec>,
-          # which this workflow promotes to the final release below.
+          # inherited from upstream (v3.4.9-424). Pre-releases do not count.
           if gh release list --repo "$GITHUB_REPOSITORY" --limit 300 \
               --json tagName,isPrerelease \
               --jq '.[] | select(.isPrerelease | not) | .tagName' \
-              | grep -Eq "^v${local_spec}$|-${local_spec}$"; then
-            echo "A release for spec_version $local_spec already exists; nothing to do."
-            echo "ready=false" >> $GITHUB_OUTPUT
+              | grep -Eq "^v${chain_spec}$|-${chain_spec}$"; then
+            echo "GitHub release for spec_version $chain_spec is complete."
+            release_needed=false
+          else
+            release_needed=true
+          fi
+
+          # v432 and earlier predate this marker. Every later
+          # release retries until PyPI succeeds and this exact marker is stored.
+          if [ "$chain_spec" -le 432 ]; then
+            echo "v432 Python publication is handled manually."
+            python_needed=false
+          else
+            marker_id=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/v${chain_spec}" \
+              --jq '.assets[] | select(.name == "python-publish-complete.json") | .id' \
+              2>/dev/null | head -n 1 || true)
+            if [ -z "$marker_id" ]; then
+              python_needed=true
+            else
+              gh api -H "Accept: application/octet-stream" \
+                "repos/$GITHUB_REPOSITORY/releases/assets/$marker_id" \
+                > /tmp/python-publish-complete.json
+              tag_sha=$(gh api "repos/$GITHUB_REPOSITORY/commits/v${chain_spec}" --jq '.sha')
+              jq -e --arg sha "$tag_sha" --argjson spec "$chain_spec" \
+                '.schema == 1 and .spec_version == $spec and .commit == $sha' \
+                /tmp/python-publish-complete.json >/dev/null \
+                || { echo "invalid Python publish marker for v${chain_spec}"; exit 1; }
+              echo "Stable Python packages are already published."
+              python_needed=false
+            fi
+          fi
+          echo "release_needed=$release_needed" >> "$GITHUB_OUTPUT"
+          echo "python_needed=$python_needed" >> "$GITHUB_OUTPUT"
+
+          # A valid marker is the durable terminal state. Resolve the expiring
+          # workflow artifact only while release or Python work is pending.
+          if [ "$release_needed" = false ] && [ "$python_needed" = false ]; then
             exit 0
           fi
 
-          # Use the commit recorded by the release train, not HEAD — main may
-          # have advanced with docs-only merges after the multisig proposal.
-          # Resolve the artifact through the provenance gate: fork PRs can plant
-          # a `mainnet-upgrade-<spec>` artifact in this repo's store, and this
-          # commit is what we later publish to PyPI/crates.io and push to
-          # `mainnet`, so it must come from a push of release-train.yml to main.
-          artifact_id=$(.github/scripts/resolve-release-artifact.sh "$local_spec")
-          : ${artifact_id:?no trustworthy mainnet-upgrade-${local_spec} artifact found — did the release train run?}
+          # Resolve what actually runs on chain, not HEAD: main can advance
+          # while a multisig waits for its final signature. The provenance gate
+          # admits artifacts only from release-train.yml pushes to main.
+          artifact_id=$(.github/scripts/resolve-release-artifact.sh "$chain_spec")
+          : ${artifact_id:?no trustworthy mainnet-upgrade-${chain_spec} artifact found — did the release train run?}
+          rm -rf /tmp/release-artifact /tmp/release-artifact.zip
           gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/${artifact_id}/zip" > /tmp/release-artifact.zip
           unzip -o /tmp/release-artifact.zip -d /tmp/release-artifact
           release_sha=$(jq -r '.commit' /tmp/release-artifact/pending-release.json)
           : ${release_sha:?pending-release.json missing commit field}
+          expected_spec=$(jq -er '.expected_spec_version | numbers' \
+            /tmp/release-artifact/pending-release.json)
+          [ "$expected_spec" = "$chain_spec" ] \
+            || { echo "artifact expects spec $expected_spec, chain runs $chain_spec"; exit 1; }
           echo "Release train commit: $release_sha"
-          echo "sha=$release_sha" >> $GITHUB_OUTPUT
-
-          echo "Chain is upgraded to $local_spec and no release exists yet."
-          echo "ready=true" >> $GITHUB_OUTPUT
+          echo "sha=$release_sha" >> "$GITHUB_OUTPUT"
 
   release:
     name: Cut GitHub release
     needs: check
-    if: needs.check.outputs.ready == 'true'
+    if: needs.check.outputs.release_needed == 'true'
     runs-on: [self-hosted, fireactions-turbo-8]
     # MIRROR_DEPLOY_KEY lives in the mainnet environment secrets; one
     # approval of the run covers this and the publish jobs below.
@@ -195,7 +231,7 @@ jobs:
   publish-docker:
     name: Dispatch Docker image publishing
     needs: [check, release]
-    if: needs.check.outputs.ready == 'true'
+    if: needs.check.outputs.release_needed == 'true'
     runs-on: [self-hosted, fireactions-turbo-8]
     permissions:
       actions: write
@@ -213,26 +249,30 @@ jobs:
             --repo "$GITHUB_REPOSITORY" --ref "$tag" -f branch-or-tag="$tag"
 
   # Full platform matrix (manylinux x86_64/aarch64, macOS arm64/x86_64,
-  # sdist) at the released commit; the committed version is published as-is.
+  # sdist) at the released commit; the SDK's committed .dev0 is stamped stable.
   build-core:
     name: Build bittensor-core wheels
-    needs: [check, release]
-    if: needs.check.outputs.ready == 'true'
+    needs: check
+    if: needs.check.outputs.python_needed == 'true'
     uses: ./.github/workflows/build-core-wheels.yml
     with:
       ref: ${{ needs.check.outputs.sha }}
 
   publish-sdk:
-    name: Publish Python SDK to PyPI
+    name: Publish Python packages to PyPI
     needs: [check, release, build-core]
-    if: needs.check.outputs.ready == 'true'
+    if: >-
+      always() &&
+      needs.check.outputs.python_needed == 'true' &&
+      needs.build-core.result == 'success' &&
+      (needs.release.result == 'success' || needs.release.result == 'skipped')
     runs-on: [self-hosted, fireactions-turbo-8]
     # PyPI trusted publishing (OIDC); publishers for `bittensor` and
     # bittensor-core are registered against this workflow + mainnet environment.
     environment: mainnet
     permissions:
       id-token: write
-      contents: read
+      contents: write # attach the durable completion marker to the release
     steps:
       - uses: actions/checkout@v4
         with:
@@ -250,6 +290,9 @@ jobs:
           path: dist
           merge-multiple: true
 
+      - name: Prepare stable SDK distribution
+        run: python3 .github/scripts/prepare-sdk-dist.py
+
       - name: Build SDK wheel and sdist
         working-directory: sdk/python
         run: uv build --out-dir ../../dist
@@ -271,10 +314,27 @@ jobs:
           uv publish --trusted-publishing always \
             --check-url https://pypi.org/simple/ dist/*
 
+      # This is the only durable state for Python publication. It is written
+      # after uv reports that every distribution was uploaded or already exists.
+      - name: Record successful Python publication
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_SHA: ${{ needs.check.outputs.sha }}
+          SPEC_VERSION: ${{ needs.check.outputs.spec_version }}
+        run: |
+          jq -n \
+            --arg sha "$RELEASE_SHA" \
+            --argjson spec "$SPEC_VERSION" \
+            --arg workflow_run "$GITHUB_RUN_ID" \
+            '{schema: 1, spec_version: $spec, commit: $sha, workflow_run: $workflow_run}' \
+            > python-publish-complete.json
+          gh release upload "v${SPEC_VERSION}" --repo "$GITHUB_REPOSITORY" --clobber \
+            python-publish-complete.json
+
   publish-crates:
     name: Publish Rust crates to crates.io
     needs: [check, release]
-    if: needs.check.outputs.ready == 'true'
+    if: needs.check.outputs.release_needed == 'true'
     runs-on: [self-hosted, fireactions-turbo-8]
     environment: mainnet
     steps:
@@ -310,7 +370,7 @@ jobs:
   promote-website:
     name: Deploy production website and docs
     needs: [check, release]
-    if: needs.check.outputs.ready == 'true'
+    if: needs.check.outputs.release_needed == 'true'
     runs-on: [self-hosted, fireactions-turbo-8]
     environment: mainnet
     steps:

diff --git a/docs/internals/release-process.mdx b/docs/internals/release-process.mdx
@@ -74,9 +74,9 @@ running your runtime with a public endpoint
 
 ## The release watcher
 
-`watch-mainnet-release.yml` polls mainnet every 10 minutes. When the on-chain
-`spec_version` matches the `main` branch and no GitHub release exists for it
-yet, it cuts the release train's artifacts:
+`watch-mainnet-release.yml` polls mainnet every 10 minutes. When an upgrade
+executes, it resolves the release-train artifact for the `spec_version`
+actually running on chain and cuts the release:
 
 1. **GitHub release** `v<spec_version>`
 2. **Production Docker images** tagged `v<spec_version>` and `latest`
@@ -89,6 +89,13 @@ yet, it cuts the release train's artifacts:
 This ordering means the release always reflects what is *actually running on
 mainnet*, not what was merged.
 
+Stable Python publication has an independent completion marker attached to the
+GitHub release. The marker is written only after PyPI accepts every SDK and
+`bittensor-core` distribution, so a failed or partial upload is retried on the
+next watcher run even if the GitHub release already exists. The SDK's committed
+`X.Y.Z.dev0` version is stamped to `X.Y.Z` for this build. The already-deployed
+v432 release predates the marker and is recovered manually.
+
 The watcher explicitly dispatches `docker.yml` and `docker-localnet.yml`
 because releases created with the default `GITHUB_TOKEN` do not emit another
 workflow event. The production build publishes both the immutable release tag