[CI] Add nightly build/release and test workflow

.github/workflows/nightly.yml

@@ -0,0 +1,342 @@
+name: Nightly Build & Test
+
+on:
+  schedule:
+    - cron: '0 16 * * *'  # 00:00 Beijing Time (UTC+8)
+  workflow_dispatch:
+    inputs:
+      skip_publish:
+        description: 'Skip publishing to TestPyPI'
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  issues: write
+
+concurrency:
+  group: nightly-${{ github.ref_name }}
+  cancel-in-progress: true
+
+# ── Workflow Design Notes ──────────────────────────────────────────────────────
+# 1. version-stamp computes a nightly version string for artifact naming and
+#    traceability (e.g. "0.8.0.dev20250716"). The actual wheel package version
+#    is determined by pyproject.toml inside the reusable _build-wheel.yaml.
+# 2. Build jobs use the reusable _build-wheel.yaml which does its own checkout
+#    and derives VERSION from GITHUB_REF_NAME / pyproject.toml.
+# 3. publish-testpypi uploads all nightly-* artifacts to TestPyPI.
+# 4. nightly-gate aggregates results; notify-failure files an issue on failure.
+# ──────────────────────────────────────────────────────────────────────────────
+
+env:
+  SCCACHE_GHA_ENABLED: "true"
+
+jobs:
+  version-stamp:
+    runs-on: ubuntu-22.04
+    outputs:
+      nightly_version: ${{ steps.version.outputs.nightly_version }}
+    steps:
+      - uses: actions/checkout@v4
+      # NOTE: nightly_version is used for artifact naming and traceability.
+      # Wheel package version comes from pyproject.toml via the reusable workflow.
+      # TODO: Add a `version-override` input to `_build-wheel.yaml` for proper
+      #       nightly versioning so downstream builds can stamp dev versions.
+      - name: Compute nightly version
+        id: version
+        run: |
+          BASE_VERSION=$(grep -Po '(?<=^version = ")[^"]+' mooncake-wheel/pyproject.toml)
+          NIGHTLY_VERSION="${BASE_VERSION}.dev$(date -u +%Y%m%d)"
+          echo "nightly_version=$NIGHTLY_VERSION" >> "$GITHUB_OUTPUT"
+          echo "Nightly version: $NIGHTLY_VERSION"
+
+  build-cuda128-x86:
+    needs: version-stamp
+    uses: ./.github/workflows/_build-wheel.yaml
+    with:
+      runner: ubuntu-22.04
+      container: pytorch/manylinux2_28-builder:cuda12.8
+      python-versions: '["3.10", "3.12"]'
+      cuda: container
+      build-with-ep: '1'
+      torch-cuda-arch-list: '8.0;9.0'
+      ep-torch-versions: '2.11.0;2.12.0;2.12.1'
+      build-nvlink-allocator: true
+      cmake-args: >-
+        -DBUILD_UNIT_TESTS=OFF -DUSE_HTTP=ON -DUSE_ETCD=ON -DUSE_CUDA=ON
+        -DWITH_EP=ON -DSTORE_USE_ETCD=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release
+      artifact-prefix: nightly-cuda128-x86
+
+  build-cuda128-arm64:
+    needs: version-stamp
+    uses: ./.github/workflows/_build-wheel.yaml
+    with:
+      runner: ubuntu-22.04-arm
+      python-versions: '["3.10", "3.12"]'
+      cuda: sbsa-12.8
+      cmake-generator: Ninja
+      torch-cuda-arch-list: '9.0'
+      cmake-args: >-
+        -DUSE_HTTP=ON -DUSE_CUDA=ON -DUSE_MNNVL=ON -DWITH_EP=OFF
+        -DWITH_STORE_RUST=OFF -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release
+      artifact-prefix: nightly-cuda128-arm64
+
+  build-cuda13-x86:
+    needs: version-stamp
+    uses: ./.github/workflows/_build-wheel.yaml
+    with:
+      runner: ubuntu-22.04
+      container: pytorch/manylinux2_28-builder:cuda13.0
+      python-versions: '["3.10", "3.12"]'
+      cuda: container
+      build-with-ep: '1'
+      variant-flag: CU13_BUILD
+      torch-cuda-arch-list: '8.0;9.0'
+      ep-torch-versions: '2.11.0;2.12.0;2.12.1'
+      build-nvlink-allocator: true
+      cmake-args: >-
+        -DBUILD_UNIT_TESTS=OFF -DUSE_HTTP=ON -DUSE_ETCD=ON -DUSE_CUDA=ON
+        -DWITH_EP=ON -DSTORE_USE_ETCD=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release
+      artifact-prefix: nightly-cuda13-x86
+
+  publish-testpypi:
+    needs: [version-stamp, build-cuda128-x86, build-cuda128-arm64, build-cuda13-x86]
+    if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.skip_publish == 'true') }}
+    runs-on: ubuntu-22.04
+    environment: nightly
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Log nightly version
+        run: |
+          echo "Publishing nightly version: ${{ needs.version-stamp.outputs.nightly_version }}"
+
+      - name: Download all nightly wheel artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist-all
+          pattern: 'nightly-*'
+
+      - name: Collect and rename wheels with nightly version
+        run: |
+          mkdir -p dist-publish
+          find dist-all -name "*.whl" -exec cp {} dist-publish/ \;
+          echo "Collected wheels:"
+          ls -la dist-publish/
+
+      - name: Install twine
+        run: pip install twine
+
+      - name: Validate wheels
+        run: twine check dist-publish/*.whl
+
+      - name: Publish to TestPyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TESTPYPI_API_TOKEN }}
+        run: |
+          twine upload --repository testpypi --skip-existing dist-publish/*.whl
+
+  nightly-test:
+    runs-on: ubuntu-22.04
+    env:
+      CI: "true"
+      SCCACHE_GHA_ENABLED: "true"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install and start etcd
+        run: |
+          wget -q https://github.com/etcd-io/etcd/releases/download/v3.6.1/etcd-v3.6.1-linux-amd64.tar.gz
+          tar xzf etcd-v3.6.1-linux-amd64.tar.gz
+          sudo mv etcd-v3.6.1-linux-amd64/etcd* /usr/local/bin/
+          etcd --advertise-client-urls http://127.0.0.1:2379 --listen-client-urls http://127.0.0.1:2379 &
+          sleep 3
+          etcdctl --endpoints=http://127.0.0.1:2379 endpoint health
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/lib/android
+          df -h
+
+      - name: Install CUDA Toolkit
+        uses: Jimver/[email protected]
+        with:
+          cuda: '12.8.1'
+          linux-local-args: '["--toolkit"]'
+          method: 'network'
+          sub-packages: '["nvcc"]'
+
+      - name: Install build utilities
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+
+      - name: Configure sccache
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Configure project (Release, no ASAN)
+        run: |
+          sudo apt update -y
+          sudo bash -x dependencies.sh -y
+          mkdir build && cd build
+          cmake -G Ninja .. \
+            -DUSE_HTTP=ON -DUSE_CXL=ON -DUSE_UB=ON -DUSE_ETCD=ON \
+            -DSTORE_USE_ETCD=ON -DCMAKE_BUILD_TYPE=Release \
+            -DBUILD_UNIT_TESTS=ON -DENABLE_SCCACHE=ON
+
+      - name: Build project
+        run: |
+          cd build
+          cmake --build . -j$(nproc)
+          sudo cmake --install .
+
+      - name: Build nvlink_allocator.so
+        run: |
+          mkdir -p build/mooncake-transfer-engine/nvlink-allocator
+          cd mooncake-transfer-engine/nvlink-allocator
+          export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
+          bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/
+
+      - name: Start Metadata Server
+        run: |
+          cd mooncake-transfer-engine/example/http-metadata-server-python
+          pip install aiohttp
+          python ./bootstrap_server.py &
+          sleep 2
+
+      - name: Run CTest unit tests
+        run: |
+          cd build
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+          MC_METADATA_SERVER=http://127.0.0.1:8080/metadata \
+          DEFAULT_KV_LEASE_TTL=500 \
+            ctest -j --output-on-failure
+
+      - name: Run Mooncake Store Rust smoke test
+        run: |
+          $GITHUB_WORKSPACE/build/mooncake-store/src/mooncake_master \
+            --eviction_high_watermark_ratio=0.95 \
+            --cluster_id=nightly_rust_cluster \
+            --port 50051 &
+          MASTER_PID=$!
+          sleep 3
+          cd mooncake-store/rust
+          export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/mooncake-asio:$GITHUB_WORKSPACE/build/mooncake-store/src:$GITHUB_WORKSPACE/build/mooncake-store/src/cachelib_memory_allocator:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base:$GITHUB_WORKSPACE/build/mooncake-common/etcd:$LD_LIBRARY_PATH
+          export MOONCAKE_BUILD_DIR=$GITHUB_WORKSPACE/build
+          export MOONCAKE_STORE_LIB_DIR=$GITHUB_WORKSPACE/build/mooncake-store/src
+          export MOONCAKE_STORE_INCLUDE_DIR=$GITHUB_WORKSPACE/mooncake-store/include
+          export MC_METADATA_SERVER=http://127.0.0.1:8080/metadata
+          export MC_RUST_STORE_RUN_INTEGRATION=true
+          export MC_RUST_STORE_MASTER_ADDR=127.0.0.1:50051
+          export MC_RUST_STORE_LOCAL_HOSTNAME=127.0.0.1
+          export MC_RUST_STORE_PROTOCOL=tcp
+          export MC_RUST_STORE_DEVICE_NAME=
+          cargo test --test minimal_smoke -- --nocapture
+          MC_RUST_BENCH_ITERATIONS=4 \
+          MC_RUST_BENCH_VALUE_SIZE=4096 \
+          MC_RUST_BENCH_WARMUP=1 \
+            cargo run --release --example store_benchmark
+          kill $MASTER_PID 2>/dev/null || true
+
+      - name: Run Go store binding integration tests
+        run: |
+          $GITHUB_WORKSPACE/build/mooncake-store/src/mooncake_master \
+            --eviction_high_watermark_ratio=0.95 \
+            --cluster_id=nightly_go_cluster \
+            --port 50051 &
+          MASTER_PID=$!
+          sleep 3
+          cd mooncake-store/go
+          export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/mooncake-common:$GITHUB_WORKSPACE/build/mooncake-store/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base:$GITHUB_WORKSPACE/build/mooncake-common/etcd
+          export CGO_ENABLED=1
+          export CGO_CFLAGS="-I$GITHUB_WORKSPACE/mooncake-store/include -I$GITHUB_WORKSPACE/mooncake-transfer-engine/include"
+          export CGO_LDFLAGS="-L$GITHUB_WORKSPACE/build/mooncake-store/src -L$GITHUB_WORKSPACE/build/mooncake-store/src/cachelib_memory_allocator -L$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src -L$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base -L$GITHUB_WORKSPACE/build/mooncake-common -L$GITHUB_WORKSPACE/build/mooncake-common/etcd -lmooncake_store -lcachelib_memory_allocator -ltransfer_engine -lbase -lasio -letcd_wrapper -lstdc++ -lnuma -lglog -lgflags -libverbs -lmlx5 -ljsoncpp -lzstd -lcurl -luring -lm -lxxhash -lyaml-cpp"
+          if [ -d /usr/local/cuda/lib64 ]; then export CGO_LDFLAGS="$CGO_LDFLAGS -L/usr/local/cuda/lib64 -lcudart"; fi
+          if ldconfig -p 2>/dev/null | grep -q libzmq; then export CGO_LDFLAGS="$CGO_LDFLAGS -lzmq"; fi
+          MC_METADATA_SERVER=http://127.0.0.1:8080/metadata go test -v ./tests/...
+          kill $MASTER_PID 2>/dev/null || true
+
+      - name: Build and install Python wheel for integration tests
+        run: |
+          export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
+          export CUDA_HOME=/usr/local/cuda
+          PYTHON_VERSION=3.12 OUTPUT_DIR=dist ./scripts/build_wheel.sh
+          pip install mooncake-wheel/dist/*.whl
+
+      - name: Run Python integration tests (full suite)
+        env:
+          MC_METADATA_SERVER: http://127.0.0.1:8080/metadata
+          DEFAULT_KV_LEASE_TTL: "500"
+          TEST_SSD_OFFLOAD_IN_EVICT: "1"
+          TEST_PROMOTION_ON_HIT: "1"
+        run: |
+          export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
+          bash scripts/run_tests.sh
+
+  nightly-gate:
+    if: always()
+    needs: [publish-testpypi, nightly-test]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check results
+        run: |
+          echo "=== Nightly Gate Summary ==="
+          echo "Publish: ${{ needs.publish-testpypi.result }}"
+          echo "Test: ${{ needs.nightly-test.result }}"
+          if [ "${{ needs.nightly-test.result }}" != "success" ]; then
+            echo "::error::Nightly tests failed!"
+            exit 1
+          fi
+          if [ "${{ needs.publish-testpypi.result }}" == "failure" ]; then
+            echo "::error::Nightly publish failed!"
+            exit 1
+          fi
+          echo "All nightly jobs passed!"
+
+  notify-failure:
+    if: ${{ always() && needs.nightly-gate.result == 'failure' }}
+    needs: [nightly-gate]
+    runs-on: ubuntu-22.04
+    permissions:
+      issues: write
+    steps:
+      - name: Create failure issue
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const title = `Nightly build/test failure - ${new Date().toISOString().slice(0, 10)}`;
+            const body = [
+              '## Nightly Failure Report',
+              '',
+              `**Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+              `**Branch**: ${context.ref}`,
+              `**Timestamp**: ${new Date().toISOString()}`,
+              '',
+              'Please investigate the failed jobs in the workflow run linked above.',
+            ].join('\n');
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title,
+              body,
+              labels: ['nightly-failure'],
+            });