validate-orchestrator-integration.yml

name: Orchestrator Integration Tests

# ==============================================================================
# Exhaustive integration tests — runs on a daily cron and manual dispatch.
# Slow (~1-2h wall-clock): k8s, AWS, local-docker, rclone via MiniStack + k3d.
#
# Mirrors the full orchestrator-integrity.yml test suite from the orchestrator
# repo, run from unity-builder's perspective to catch cross-repo regressions.
#
# For fast per-PR checks, see validate-orchestrator.yml.
# ==============================================================================

on:
  workflow_dispatch:
  workflow_call: # Allow integrity-check.yml and other workflows to invoke these tests
  schedule:
    - cron: '0 3 * * *' # Daily at 3 AM UTC

permissions:
  contents: read
  checks: write
  statuses: write

# Note: no concurrency block here — when invoked via workflow_call, the caller
# (integrity-check.yml) manages concurrency. For direct dispatch/cron, runs are
# naturally serialized by GitHub's single-concurrency-per-ref default.

env:
  AWS_STACK_NAME: game-ci-team-pipelines
  DEBUG: true
  PROJECT_PATH: test-project
  USE_IL2CPP: false
  # Integration tests validate credential transport only; real licensing secrets are prohibited.
  UNITY_EMAIL: unity-builder-integration@example.invalid
  UNITY_PASSWORD: integration-test-only
  UNITY_SERIAL: integration-test-only

# ==============================================================================
# 4 parallel jobs on separate runners (fresh 14GB disk each).
# Matches the orchestrator-integrity.yml architecture.
# ==============================================================================

jobs:
  # ============================================================================
  # PLUGIN INTERFACE (fast gate — fails fast before slow jobs waste time)
  # ============================================================================
  plugin-interface:
    name: Plugin Interface Tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout unity-builder
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5

      - name: Checkout orchestrator (matching branch)
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          ref: ${{ github.head_ref || github.ref_name }}
          path: orchestrator-standalone
        continue-on-error: true
        id: orchestrator-branch

      - name: Fallback to orchestrator main branch
        if: steps.orchestrator-branch.outcome == 'failure'
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          path: orchestrator-standalone

      - name: Install package manager (from package.json)
        run: |
          corepack enable
          corepack install
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
        with:
          node-version: 20
      - name: Resolve yarn cache folder
        id: yarn-config
        run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn install cache (node_modules + cacheFolder + install-state)
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: |
            ${{ steps.yarn-config.outputs.cacheFolder }}
            .yarn/install-state.gz
          key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            yarn-v2-${{ runner.os }}-node-20-

      - name: Install unity-builder dependencies
        run: yarn install --immutable

      - name: Build unity-builder
        run: |
          echo "Building unity-builder TypeScript..."
          npx tsc
          echo "✓ unity-builder compiles successfully"

      - name: Run plugin interface unit tests
        run: |
          echo "Running orchestrator-plugin unit tests..."
          npx vitest run orchestrator-plugin --reporter=verbose

      - name: Build and pack orchestrator
        working-directory: orchestrator-standalone
        run: |
          yarn install --immutable
          echo "Building orchestrator..."
          npx tsc
          echo "✓ orchestrator compiles successfully"
          echo "Packing orchestrator as tarball..."
          npm pack

      - name: Install orchestrator into unity-builder
        run: |
          echo "Installing orchestrator into unity-builder workspace..."
          npm install ./orchestrator-standalone/game-ci-orchestrator-*.tgz --no-save --legacy-peer-deps

      - name: Verify plugin loader returns exports with orchestrator installed
        run: |
          node -e "
            const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
            (async () => {
              const plugin = await loadOrchestratorPlugin();
              if (plugin === undefined) {
                console.error('ERROR: loadOrchestratorPlugin should return defined plugin when package is installed');
                process.exit(1);
              }
              const lifecycleMethods = [
                'initialize', 'canHandleBuild', 'handleBuild',
                'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild',
              ];
              for (const method of lifecycleMethods) {
                if (typeof plugin[method] !== 'function') {
                  console.error('ERROR: plugin.' + method + ' should be a function, got ' + typeof plugin[method]);
                  process.exit(1);
                }
              }
              console.log('✓ loadOrchestratorPlugin() returns plugin with all ' + lifecycleMethods.length + ' lifecycle methods');
            })();
          "

      - name: Verify type declarations match exports
        run: |
          node -e "
            const orch = require('@game-ci/orchestrator');
            const expected = ['Orchestrator','BuildReliabilityService','TestWorkflowService','HotRunnerService','OutputService','OutputTypeRegistry','ArtifactUploadHandler','IncrementalSyncService','ChildWorkspaceService','LocalCacheService','SubmoduleProfileService','LfsAgentService','GitHooksService'];
            const missing = expected.filter(e => orch[e] === undefined);
            if (missing.length > 0) { console.error('Missing exports:', missing.join(', ')); process.exit(1); }
            console.log('✓ All ' + expected.length + ' exports present');
          "

  # ============================================================================
  # K8S INTEGRATION TESTS (k3d + MiniStack)
  # ============================================================================
  k8s-integration:
    name: K8s Integration Tests
    runs-on: ubuntu-latest
    env:
      K3D_NODE_CONTAINERS: 'k3d-unity-builder-agent-0'
      AWS_FORCE_PROVIDER: aws-local
      RESOURCE_TRACKING: 'true'
      K8S_LOCALSTACK_HOST: localstack-main
    steps:
      - name: Checkout orchestrator (matching branch)
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          ref: ${{ github.head_ref || github.ref_name }}
          lfs: false
        continue-on-error: true
        id: orch-branch

      - name: Clean corrupted checkout
        if: steps.orch-branch.outcome == 'failure'
        run: rm -rf .git || true

      - name: Fallback to orchestrator main branch
        if: steps.orch-branch.outcome == 'failure'
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Install package manager (from package.json)
        run: |
          corepack enable
          corepack install
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
        with:
          node-version: 20
      - name: Resolve yarn cache folder
        id: yarn-config
        run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn install cache (node_modules + cacheFolder + install-state)
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: |
            ${{ steps.yarn-config.outputs.cacheFolder }}
            .yarn/install-state.gz
          key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            yarn-v2-${{ runner.os }}-node-20-

      - name: Set up kubectl
        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede
        with:
          version: 'v1.34.1'

      - name: Install k3d
        run: |
          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
          k3d version | cat

      - name: Define cleanup functions
        run: |
          cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF'
          light_cleanup() {
            echo "--- Light cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -f || true
            df -h
          }

          k8s_resource_cleanup() {
            echo "--- K8s resource cleanup ---"
            kubectl delete jobs --all --ignore-not-found=true -n default || true
            kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
              kubectl delete "$pod" --ignore-not-found=true || true
            done || true
            kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
              kubectl delete "$pvc" --ignore-not-found=true || true
            done || true
            kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
              kubectl delete "$secret" --ignore-not-found=true || true
            done || true
          }

          k3d_node_cleanup() {
            echo "--- K3d node image cleanup (preserving Unity images) ---"
            K3D_NODE_CONTAINERS="${K3D_NODE_CONTAINERS:-k3d-unity-builder-agent-0 k3d-unity-builder-server-0}"
            for NODE in $K3D_NODE_CONTAINERS; do
              docker exec "$NODE" sh -c "crictl rm --all 2>/dev/null || true" || true
              docker exec "$NODE" sh -c "for img in \$(crictl images -q 2>/dev/null); do repo=\$(crictl inspecti \$img --format '{{.repo}}' 2>/dev/null || echo ''); if echo \"\$repo\" | grep -qvE 'unityci/editor|unity'; then crictl rmi \$img 2>/dev/null || true; fi; done" || true
              docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
            done || true
          }

          full_k8s_cleanup() {
            k8s_resource_cleanup
            k3d_node_cleanup
            light_cleanup
          }
          CLEANUP_EOF
          echo "Cleanup functions defined at /tmp/cleanup-functions.sh"

      - name: Initial disk space cleanup
        run: |
          echo "Initial disk space cleanup..."
          df -h
          k3d cluster delete unity-builder || true
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          docker network rm orchestrator-net 2>/dev/null || true
          docker network create orchestrator-net || true
          echo "Disk usage after cleanup:"
          df -h

      - name: Start MiniStack
        run: |
          echo "Starting MiniStack..."
          docker run -d \
            --name localstack-main \
            --network orchestrator-net \
            --add-host=host.docker.internal:host-gateway \
            -p 4566:4566 \
            ministackorg/ministack:latest || true
          echo "Waiting for MiniStack to be ready..."
          MAX_ATTEMPTS=60
          READY=false
          for i in $(seq 1 $MAX_ATTEMPTS); do
            if ! docker ps | grep -q localstack-main; then sleep 2; continue; fi
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if [ -z "$HEALTH" ] || ! echo "$HEALTH" | grep -q "services"; then sleep 2; continue; fi
            if echo "$HEALTH" | grep -q '"s3"'; then
              echo "MiniStack is ready (attempt $i/$MAX_ATTEMPTS)"
              READY=true
              break
            fi
            sleep 2
          done
          if [ "$READY" != "true" ]; then
            echo "ERROR: MiniStack did not become ready"
            docker logs localstack-main --tail 100 || true
            exit 1
          fi

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi
          pip install awscli-local || true

      - name: Create S3 bucket for tests
        run: |
          for _ in {1..10}; do
            if curl -s http://localhost:4566/_localstack/health > /dev/null 2>&1; then break; fi
            sleep 1
          done
          for _ in {1..5}; do
            if command -v awslocal > /dev/null 2>&1; then
              awslocal s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            else
              aws --endpoint-url=http://localhost:4566 s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            fi
            sleep 2
          done

      - run: yarn install --immutable

      - name: Build orchestrator
        run: |
          echo "Building orchestrator TypeScript..."
          yarn build
          echo "✓ orchestrator build successful"

      # --- Fast unit tests (fast-fail gate) ---
      - name: Run orchestrator unit tests (fast, no infra)
        timeout-minutes: 2
        run: >-
          yarn vitest run
          "orchestrator-guid"
          "orchestrator-folders"
          "task-parameter-serializer"
          "follow-log-stream-service"
          "runner-availability-service"
          "provider-url-parser"
          "provider-loader"
          "provider-git-manager"
          "orchestrator-image"
          "orchestrator-hooks"
          "orchestrator-github-checks"
          "middleware-service"
          --reporter=verbose --no-file-parallelism

      # --- K8s cluster setup ---
      - name: Clean up disk space before K8s tests
        run: |
          rm -rf ./orchestrator-cache/* || true
          sudo apt-get clean || true
          docker system prune -f || true
          df -h

      - name: Create k3s cluster (k3d)
        timeout-minutes: 5
        run: |
          LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "")
          echo "MiniStack container IP: $LOCALSTACK_IP"
          k3d cluster create unity-builder \
            --agents 1 \
            --network orchestrator-net \
            --wait
          kubectl config current-context | cat
          echo "LOCALSTACK_IP=$LOCALSTACK_IP" >> "$GITHUB_ENV"

      - name: Verify cluster readiness and MiniStack connectivity
        timeout-minutes: 2
        run: |
          for i in {1..60}; do
            if kubectl get nodes 2>/dev/null | grep -q Ready; then echo "Cluster is ready"; break; fi
            echo "Waiting for cluster... ($i/60)"
            sleep 5
          done
          kubectl get nodes
          LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "")
          kubectl run test-localstack --image=curlimages/curl --rm -i --restart=Never --timeout=30s -- \
            curl -v --max-time 10 http://"${LOCALSTACK_IP}":4566/_localstack/health 2>&1 | head -30 || \
            echo "Cluster connectivity test - MiniStack may not be accessible from k3d"

      - name: Clean up K8s resources before tests
        run: |
          source /tmp/cleanup-functions.sh
          k8s_resource_cleanup
          for _ in {1..30}; do
            PVC_COUNT=$(kubectl get pvc -n default 2>/dev/null | grep -c "unity-builder-pvc-" || echo "0")
            if [ "$PVC_COUNT" -eq 0 ]; then echo "All PVCs deleted"; break; fi
            sleep 1
          done
          docker system prune -f || true

      # --- K8s Test 1: orchestrator-image ---
      - name: Run orchestrator-image test (K8s)
        timeout-minutes: 10
        run: yarn run test "orchestrator-image" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '512'
          containerMemory: '512'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-image (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 2: orchestrator-kubernetes ---
      - name: Run orchestrator-kubernetes test
        timeout-minutes: 30
        run: yarn run test "orchestrator-kubernetes" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-kubernetes
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 3: orchestrator-s3-steps ---
      - name: Run orchestrator-s3-steps test (K8s)
        timeout-minutes: 30
        run: yarn run test "orchestrator-s3-steps" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-s3-steps (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 4: orchestrator-end2end-caching ---
      - name: Run orchestrator-end2end-caching test (K8s)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-caching" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-caching (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 5: orchestrator-end2end-retaining ---
      - name: Extra disk cleanup before retaining test
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup
          docker system prune -af --volumes || true
          df -h
      - name: Run orchestrator-end2end-retaining test (K8s)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-retaining" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-retaining (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s teardown ---
      - name: Delete k3d cluster and final cleanup
        if: always()
        run: |
          k3d cluster delete unity-builder || true
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          df -h

  # ============================================================================
  # AWS/LOCALSTACK INTEGRATION TESTS
  # ============================================================================
  aws-integration:
    name: AWS Integration Tests
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ENDPOINT_URL: http://localhost:4566
    steps:
      - name: Checkout orchestrator (matching branch)
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          ref: ${{ github.head_ref || github.ref_name }}
          lfs: false
        continue-on-error: true
        id: orch-branch

      - name: Clean corrupted checkout
        if: steps.orch-branch.outcome == 'failure'
        run: rm -rf .git || true

      - name: Fallback to orchestrator main branch
        if: steps.orch-branch.outcome == 'failure'
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Install package manager (from package.json)
        run: |
          corepack enable
          corepack install
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
        with:
          node-version: 20
      - name: Resolve yarn cache folder
        id: yarn-config
        run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn install cache (node_modules + cacheFolder + install-state)
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: |
            ${{ steps.yarn-config.outputs.cacheFolder }}
            .yarn/install-state.gz
          key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            yarn-v2-${{ runner.os }}-node-20-

      - name: Define cleanup functions
        run: |
          cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF'
          light_cleanup() {
            echo "--- Light cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -f || true
            df -h
          }
          heavy_cleanup() {
            echo "--- Heavy cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -af --volumes || true
            df -h
          }
          CLEANUP_EOF

      - name: Initial disk space cleanup
        run: |
          df -h
          docker system prune -af --volumes || true
          df -h

      - name: Start MiniStack
        run: |
          docker run -d \
            --name localstack-main \
            -p 4566:4566 \
            ministackorg/ministack:latest || true
          MAX_ATTEMPTS=60
          for i in $(seq 1 $MAX_ATTEMPTS); do
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi
            sleep 2
          done

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi
          pip install awscli-local || true

      - name: Create S3 bucket for tests
        run: |
          for _ in {1..5}; do
            if command -v awslocal > /dev/null 2>&1; then
              awslocal s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            else
              aws --endpoint-url=http://localhost:4566 s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            fi
            sleep 2
          done

      - run: yarn install --immutable

      - name: Build orchestrator
        run: |
          echo "Building orchestrator TypeScript..."
          yarn build
          echo "✓ orchestrator build successful"

      # --- AWS Test 1: orchestrator-image ---
      - name: Run orchestrator-image test (AWS)
        timeout-minutes: 10
        run: yarn run test "orchestrator-image" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-image (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- AWS Test 2: orchestrator-environment ---
      - name: Run orchestrator-environment test (AWS)
        timeout-minutes: 30
        run: yarn run test "orchestrator-environment" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-environment (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- AWS Test 3: orchestrator-s3-steps ---
      - name: Run orchestrator-s3-steps test (AWS)
        timeout-minutes: 30
        run: yarn run test "orchestrator-s3-steps" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-s3-steps (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- AWS Test 4: orchestrator-hooks ---
      - name: Run orchestrator-hooks test (AWS)
        timeout-minutes: 30
        run: yarn run test "orchestrator-hooks" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-hooks (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- AWS Test 5: orchestrator-caching ---
      - name: Run orchestrator-caching test (AWS)
        timeout-minutes: 60
        run: yarn run test "orchestrator-caching" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-caching (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- AWS Test 6: orchestrator-locking-core ---
      - name: Run orchestrator-locking-core test (AWS)
        timeout-minutes: 60
        run: yarn run test "orchestrator-locking-core" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-locking-core (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- AWS Test 7: orchestrator-locking-get-locked ---
      - name: Run orchestrator-locking-get-locked test (AWS)
        timeout-minutes: 60
        run: yarn run test "orchestrator-locking-get-locked" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-locking-get-locked (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      # End-to-end tests (continue-on-error) — run LAST to prevent
      # workspace corruption from affecting mandatory tests above.
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

      # --- AWS Test 8: orchestrator-end2end-caching ---
      - name: Run orchestrator-end2end-caching test (AWS)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-caching" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-caching (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- AWS Test 9: orchestrator-end2end-retaining ---
      - name: Run orchestrator-end2end-retaining test (AWS)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-retaining" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-retaining (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- AWS Test 10: orchestrator-end2end-locking ---
      - name: Run orchestrator-end2end-locking test (AWS)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-locking" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-locking (AWS)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- Final cleanup ---
      - name: Final cleanup
        if: always()
        run: |
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          df -h

  # ============================================================================
  # LOCAL-DOCKER INTEGRATION TESTS
  # ============================================================================
  local-docker-integration:
    name: Local Docker Integration Tests
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ENDPOINT_URL: http://localhost:4566
    steps:
      - name: Checkout orchestrator (matching branch)
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          ref: ${{ github.head_ref || github.ref_name }}
          lfs: false
        continue-on-error: true
        id: orch-branch

      - name: Clean corrupted checkout
        if: steps.orch-branch.outcome == 'failure'
        run: rm -rf .git || true

      - name: Fallback to orchestrator main branch
        if: steps.orch-branch.outcome == 'failure'
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Install package manager (from package.json)
        run: |
          corepack enable
          corepack install
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
        with:
          node-version: 20
      - name: Resolve yarn cache folder
        id: yarn-config
        run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn install cache (node_modules + cacheFolder + install-state)
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: |
            ${{ steps.yarn-config.outputs.cacheFolder }}
            .yarn/install-state.gz
          key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            yarn-v2-${{ runner.os }}-node-20-

      - name: Define cleanup functions
        run: |
          cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF'
          light_cleanup() {
            echo "--- Light cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -f || true
            df -h
          }
          heavy_cleanup() {
            echo "--- Heavy cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -af --volumes || true
            df -h
          }
          CLEANUP_EOF

      - name: Initial disk space cleanup
        run: |
          df -h
          docker system prune -af --volumes || true
          df -h

      - name: Start MiniStack (for S3-dependent tests)
        run: |
          docker run -d \
            --name localstack-main \
            -p 4566:4566 \
            ministackorg/ministack:latest || true
          MAX_ATTEMPTS=60
          for i in $(seq 1 $MAX_ATTEMPTS); do
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi
            sleep 2
          done

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi
          pip install awscli-local || true

      - name: Create S3 bucket for tests
        run: |
          for _ in {1..5}; do
            if command -v awslocal > /dev/null 2>&1; then
              awslocal s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            else
              aws --endpoint-url=http://localhost:4566 s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            fi
            sleep 2
          done

      - run: yarn install --immutable

      - name: Build orchestrator
        run: |
          yarn build
          echo "✓ orchestrator build successful"

      # --- Local Docker Test 1: orchestrator-image ---
      - name: Run orchestrator-image test (local-docker)
        timeout-minutes: 10
        run: yarn run test "orchestrator-image" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-image (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 2: orchestrator-hooks ---
      - name: Run orchestrator-hooks test (local-docker)
        timeout-minutes: 30
        run: yarn run test "orchestrator-hooks" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-hooks (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 3: orchestrator-local-persistence ---
      - name: Run orchestrator-local-persistence test (local-docker)
        timeout-minutes: 30
        run: yarn run test "orchestrator-local-persistence" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-local-persistence (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 4: orchestrator-caching ---
      - name: Run orchestrator-caching test (local-docker)
        timeout-minutes: 30
        run: yarn run test "orchestrator-caching" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-caching (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 5: orchestrator-github-checks ---
      - name: Run orchestrator-github-checks test (local-docker)
        timeout-minutes: 30
        run: yarn run test "orchestrator-github-checks" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-github-checks (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 6: orchestrator-locking-core (with S3) ---
      - name: Run orchestrator-locking-core test (local-docker + S3)
        timeout-minutes: 30
        run: yarn run test "orchestrator-locking-core" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-locking-core (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 7: orchestrator-locking-get-locked (with S3) ---
      - name: Run orchestrator-locking-get-locked test (local-docker + S3)
        timeout-minutes: 30
        run: yarn run test "orchestrator-locking-get-locked" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-locking-get-locked (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && light_cleanup

      # --- Local Docker Test 8: orchestrator-s3-steps (with S3) ---
      - name: Run orchestrator-s3-steps test (local-docker + S3)
        timeout-minutes: 30
        run: yarn run test "orchestrator-s3-steps" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-s3-steps (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- Local Docker Test 9: orchestrator-end2end-caching (with S3) ---
      - name: Run orchestrator-end2end-caching test (local-docker + S3)
        timeout-minutes: 60
        continue-on-error: true
        run: yarn run test "orchestrator-end2end-caching" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-caching (local-docker)
        if: always()
        run: source /tmp/cleanup-functions.sh && heavy_cleanup

      # --- Final cleanup ---
      - name: Final cleanup
        if: always()
        run: |
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          df -h

  # ============================================================================
  # RCLONE INTEGRATION TESTS
  # ============================================================================
  rclone-integration:
    name: Rclone Integration Tests
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ENDPOINT_URL: http://localhost:4566
    steps:
      - name: Checkout orchestrator (matching branch)
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          ref: ${{ github.head_ref || github.ref_name }}
          lfs: false
        continue-on-error: true
        id: orch-branch

      - name: Clean corrupted checkout
        if: steps.orch-branch.outcome == 'failure'
        run: rm -rf .git || true

      - name: Fallback to orchestrator main branch
        if: steps.orch-branch.outcome == 'failure'
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Install package manager (from package.json)
        run: |
          corepack enable
          corepack install
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
        with:
          node-version: 20
      - name: Resolve yarn cache folder
        id: yarn-config
        run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn install cache (node_modules + cacheFolder + install-state)
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: |
            ${{ steps.yarn-config.outputs.cacheFolder }}
            .yarn/install-state.gz
          key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            yarn-v2-${{ runner.os }}-node-20-

      - name: Initial disk space cleanup
        run: |
          docker system prune -af --volumes || true
          df -h

      - name: Start MiniStack
        run: |
          docker run -d \
            --name localstack-main \
            -p 4566:4566 \
            ministackorg/ministack:latest || true
          MAX_ATTEMPTS=60
          for i in $(seq 1 $MAX_ATTEMPTS); do
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi
            sleep 2
          done

      - name: Install rclone
        run: |
          curl https://rclone.org/install.sh | sudo bash || true
          rclone version || echo "rclone not available"

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi
          pip install awscli-local || true

      - name: Create S3 bucket for tests
        run: |
          for _ in {1..5}; do
            if command -v awslocal > /dev/null 2>&1; then
              awslocal s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            else
              aws --endpoint-url=http://localhost:4566 s3 mb s3://"$AWS_STACK_NAME" 2>&1 && break
            fi
            sleep 2
          done

      - run: yarn install --immutable

      - name: Build orchestrator
        run: |
          yarn build
          echo "✓ orchestrator build successful"

      # --- Rclone Test ---
      - name: Run orchestrator-rclone-steps test
        timeout-minutes: 30
        run: yarn run test "orchestrator-rclone-steps" --no-file-parallelism
        env:
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          PROVIDER_STRATEGY: local-docker
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}

      # --- Final cleanup ---
      - name: Final cleanup
        if: always()
        run: |
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          df -h