diff --git a/.github/workflows/e2e-mw-dev.yml b/.github/workflows/e2e-mw-dev.yml index 9137eb9b..24895313 100644 --- a/.github/workflows/e2e-mw-dev.yml +++ b/.github/workflows/e2e-mw-dev.yml @@ -28,7 +28,7 @@ # required-reviewer Environment — that would either block external PRs even # after approval, or force an approval click on every maintainer push.) # -# Required repo configuration (one-time, see tests/e2e-mw-dev/README.md): +# Required repo configuration (one-time, see tests/mw-dev/README.md): # vars: TS_WIF_CLIENT_ID_MW_DEV, TS_WIF_AUDIENCE_MW_DEV # secrets: AWS_ECR_PUBLISH_IAM_ROLE (already exists, used by CD), # MW_DEV_ACCOUNT_ID (mw-dev AWS account id; ARNs built from it) @@ -47,7 +47,7 @@ on: - "go.mod" - "go.sum" - "Dockerfile*" - - "tests/e2e-mw-dev/**" + - "tests/mw-dev/**" - ".github/workflows/e2e-mw-dev.yml" - ".github/workflows/_image-build.yml" workflow_dispatch: @@ -125,7 +125,7 @@ jobs: go-version-file: go.mod - name: Test e2e harness scripts - run: go test -count=1 ./tests/e2e-mw-dev + run: go test -count=1 ./tests/mw-dev - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 @@ -160,17 +160,17 @@ jobs: run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region us-east-1 --alias "$KUBE_CONTEXT" - name: Deploy isolated namespace - run: bash tests/e2e-mw-dev/run.sh deploy + run: bash tests/mw-dev/run.sh deploy - name: Run e2e harness (in-cluster Job) - run: bash tests/e2e-mw-dev/run.sh test + run: bash tests/mw-dev/run.sh test-e2e # Diagnostics are only worth collecting (and only get read) when the # harness failed — on green runs this step just added ~15s to the gating # check. failure() also covers a failed deploy step. - name: Collect diagnostics if: failure() - run: bash tests/e2e-mw-dev/run.sh diagnostics + run: bash tests/mw-dev/run.sh diagnostics # Teardown runs as its OWN job so the gating `e2e` check completes the moment # the harness verdict is known instead of waiting ~1min for deprovision + @@ -214,7 +214,7 @@ jobs: # Deprovision the ci-pr ducklings (so no S3 / cnpg role+db leaks on # shared infra) then delete the namespace. - name: Teardown - run: bash tests/e2e-mw-dev/run.sh teardown + run: bash tests/mw-dev/run.sh teardown # Backstop sweep of orphaned per-PR namespaces. Scheduled-trigger only (the # PR/manual triggers run build+e2e instead). Reuses the same Tailscale + e2e @@ -248,4 +248,4 @@ jobs: - name: Update kubeconfig run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region us-east-1 --alias "$KUBE_CONTEXT" - name: Sweep stale per-PR namespaces - run: bash tests/e2e-mw-dev/run.sh e2e-cleanup + run: bash tests/mw-dev/run.sh e2e-cleanup diff --git a/.github/workflows/scenario-dev.yml b/.github/workflows/scenario-dev.yml index aa2c609a..725c405e 100644 --- a/.github/workflows/scenario-dev.yml +++ b/.github/workflows/scenario-dev.yml @@ -8,11 +8,6 @@ on: required: true default: false type: boolean - use_shared_dev: - description: Skip isolated deploy and run against the shared dev stack - required: true - default: false - type: boolean duckgres_image: description: Existing Duckgres image to deploy instead of building this SHA required: false @@ -31,9 +26,10 @@ concurrency: jobs: scenario-runner-image: + if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow }} uses: ./.github/workflows/_image-build.yml with: - dockerfile: tests/scenario/Dockerfile + dockerfile: tests/mw-dev/scenario/Dockerfile image-name: duckgres tag: scenario-runner-${{ github.run_id }}-${{ github.run_attempt }}-arm64 platform: linux/arm64 @@ -42,7 +38,7 @@ jobs: ecr-role: ${{ secrets.AWS_ECR_PUBLISH_IAM_ROLE }} duckgres-image: - if: ${{ github.event_name != 'workflow_dispatch' || (!inputs.use_shared_dev && inputs.duckgres_image == '') }} + if: ${{ github.event_name != 'workflow_dispatch' || (!inputs.skip_slow && inputs.duckgres_image == '') }} uses: ./.github/workflows/_image-build.yml with: dockerfile: Dockerfile @@ -59,114 +55,45 @@ jobs: secrets: ecr-role: ${{ secrets.AWS_ECR_PUBLISH_IAM_ROLE }} - scenarios: + scenario-full: needs: [scenario-runner-image, duckgres-image] - if: ${{ always() && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && (inputs.use_shared_dev || inputs.duckgres_image != ''))) }} + if: ${{ always() && (github.event_name != 'workflow_dispatch' || !inputs.skip_slow) && needs.scenario-runner-image.result == 'success' && (needs.duckgres-image.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.duckgres_image != '')) }} runs-on: ubuntu-24.04 timeout-minutes: 270 - strategy: - fail-fast: false - max-parallel: 1 - matrix: - include: - - name: provision-smoke - slot: 1 - file: tests/scenario/scenarios/provision_smoke.yaml - max_runtime: 30m - go_test_timeout: 45m - slow: false - needs_dbt: false - - name: provision-rejection - slot: 2 - file: tests/scenario/scenarios/provision_rejection.yaml - max_runtime: 30m - go_test_timeout: 45m - slow: false - needs_dbt: false - - name: frozen-metadata - slot: 3 - file: tests/scenario/scenarios/posthog_frozen_metadata.yaml - max_runtime: 45m - go_test_timeout: 60m - slow: false - needs_dbt: false - - name: frozen-perf - slot: 4 - file: tests/scenario/scenarios/posthog_frozen_perf.yaml - max_runtime: 45m - go_test_timeout: 60m - slow: false - needs_dbt: false - - name: frozen-dbt - slot: 5 - file: tests/scenario/scenarios/posthog_frozen_dbt.yaml - max_runtime: 4h - go_test_timeout: 4h15m - slow: true - needs_dbt: true env: - USE_SHARED_DEV: ${{ github.event_name == 'workflow_dispatch' && inputs.use_shared_dev }} SCENARIO_RUNNER_IMAGE: ${{ needs.scenario-runner-image.outputs.image }} WORKER_IMAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.duckgres_image) || needs.duckgres-image.outputs.image }} CONTROLPLANE_IMAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.duckgres_image) || needs.duckgres-image.outputs.image }} - KUBE_CONTEXT: ${{ vars.DUCKGRES_SCENARIO_KUBE_CONTEXT }} - CLUSTER_NAME: ${{ vars.DUCKGRES_SCENARIO_EKS_CLUSTER_NAME }} - EKS_CLUSTER_NAME: ${{ vars.DUCKGRES_SCENARIO_EKS_CLUSTER_NAME }} - AWS_REGION: ${{ vars.DUCKGRES_SCENARIO_AWS_REGION || 'us-east-1' }} - CP_POD_IDENTITY_ROLE: ${{ secrets.DUCKGRES_SCENARIO_CP_POD_IDENTITY_ROLE }} - PR_NUMBER: ${{ github.run_id }}${{ matrix.slot }} - NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.slot }} - SCENARIO_NAMESPACE: duckgres-ci-pr-${{ github.run_id }}${{ matrix.slot }} - SCENARIO_NAME: ${{ matrix.name }} - SCENARIO_FILE: ${{ matrix.file }} - SCENARIO_SHARED_NAMESPACE: ${{ vars.DUCKGRES_SCENARIO_SHARED_NAMESPACE }} - SCENARIO_SHARED_API_BASE: ${{ vars.DUCKGRES_SCENARIO_SHARED_API_BASE }} - SCENARIO_SHARED_PG_HOST: ${{ vars.DUCKGRES_SCENARIO_SHARED_PG_HOST }} - SCENARIO_SHARED_FLIGHT_ADDR: ${{ vars.DUCKGRES_SCENARIO_SHARED_FLIGHT_ADDR }} - SCENARIO_CONFIG_SECRET_NAMESPACE: ${{ vars.DUCKGRES_SCENARIO_CONFIG_SECRET_NAMESPACE }} - SCENARIO_CONFIG_SECRET_NAME: ${{ vars.DUCKGRES_SCENARIO_CONFIG_SECRET_NAME }} - SCENARIO_INTERNAL_SECRET_NAME: ${{ vars.DUCKGRES_SCENARIO_INTERNAL_SECRET_NAME }} - SCENARIO_INTERNAL_SECRET_KEY: ${{ vars.DUCKGRES_SCENARIO_INTERNAL_SECRET_KEY }} - SCENARIO_ISOLATED_SNI_SUFFIX: .ci.duckgres.local - SCENARIO_SHARED_SNI_SUFFIX: ${{ vars.DUCKGRES_SCENARIO_SHARED_SNI_SUFFIX || vars.DUCKGRES_SCENARIO_SNI_SUFFIX }} - DUCKGRES_SCENARIO_MAX_RUNTIME: ${{ matrix.max_runtime }} - DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: ${{ matrix.go_test_timeout }} - DUCKGRES_SCENARIO_RUN_ID: scenario-dev-${{ matrix.name }}-${{ github.run_id }}-${{ github.run_attempt }} + KUBE_CONTEXT: posthog-mw-dev + CLUSTER_NAME: posthog-mw-dev + EKS_CLUSTER_NAME: posthog-mw-dev + AWS_REGION: us-east-1 + CP_POD_IDENTITY_ROLE: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/duckgres-control-plane-dev + PR_NUMBER: ${{ github.run_id }} + NAMESPACE: duckgres-ci-pr-${{ github.run_id }} + DUCKGRES_SCENARIO_MAX_RUNTIME: 4h + DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: 4h15m steps: - - name: Skip slow scenario - if: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_slow && matrix.slow }} - run: echo "Skipping slow scenario because skip_slow is true." - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} - - name: Isolated dev target selected - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.use_shared_dev }} - run: echo "Deploying an isolated Duckgres stack for this scenario." - - - name: Shared dev target selected - if: ${{ github.event_name == 'workflow_dispatch' && inputs.use_shared_dev }} - run: echo "Skipping isolated deploy and targeting the shared dev stack." + - name: Scenario dev target selected + run: echo "Deploying an isolated Duckgres stack for scenario.full." - name: Set up Go - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version-file: go.mod - name: Test scenario workflow scripts - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} - run: go test -count=1 ./tests/scenario ./tests/scenario-dev ./tests/e2e-mw-dev + run: go test -count=1 ./tests/mw-dev/scenario ./tests/mw-dev - name: Configure AWS credentials (OIDC) - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: role-to-assume: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/github-duckgres-e2e aws-region: us-east-1 - name: Connect to Tailscale - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 with: oauth-client-id: ${{ vars.TS_WIF_CLIENT_ID_MW_DEV }} @@ -174,34 +101,36 @@ jobs: tags: tag:github-runner - name: Install kubectl - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4.0.1 - name: Update kubeconfig - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region "$AWS_REGION" --alias "$KUBE_CONTEXT" - name: Deploy isolated Duckgres stack - if: ${{ (github.event_name != 'workflow_dispatch' || !inputs.use_shared_dev) && (github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow) }} - run: tests/scenario-dev/run.sh deploy + run: tests/mw-dev/run.sh deploy - - name: Run scenario - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }} - run: tests/scenario-dev/run.sh test + - name: Run scenario.full + run: tests/mw-dev/run.sh test-scenario-full - name: Collect diagnostics - if: ${{ failure() && (github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow) }} - run: tests/scenario-dev/run.sh diagnostics + if: failure() + run: tests/mw-dev/run.sh diagnostics - name: Teardown - if: ${{ always() && (github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow) }} - run: tests/scenario-dev/run.sh teardown + if: always() + run: tests/mw-dev/run.sh teardown - name: Upload scenario artifacts - if: ${{ always() && (github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow) }} + if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: scenario-dev-${{ matrix.name }}-${{ github.run_id }}-${{ github.run_attempt }} + name: scenario-dev-full-${{ github.run_id }}-${{ github.run_attempt }} path: artifacts/scenario-dev/ if-no-files-found: warn retention-days: 14 + + scenario-skipped: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_slow }} + runs-on: ubuntu-24.04 + steps: + - run: echo "Skipping scenario.full because skip_slow is true." diff --git a/docs/design/admin-ui.md b/docs/design/admin-ui.md index 68d0a0f4..4b8019e1 100644 --- a/docs/design/admin-ui.md +++ b/docs/design/admin-ui.md @@ -155,6 +155,6 @@ and the admin Google-group name via values. Ship to `mw-dev` first, then `mw-pro Unit (`-tags kubernetes`): RBAC gate (viewer blocked from writes/impersonation; TokenSet bypass), Prometheus proxy allow-list, impersonation round-trip with a fake stack + session-destroy-on-defer, audit row written per mutation, new configstore writes. -e2e (`tests/e2e-mw-dev/harness.sh`): admin UI reachable in-cluster, RBAC enforced, +e2e (`tests/mw-dev/e2e/harness.sh`): admin UI reachable in-cluster, RBAC enforced, impersonation SQL round-trip against a real org worker is audited, a config-store edit via the API is observed by a subsequent connection. Update CLAUDE.md + admin/README.md. diff --git a/docs/design/billing-compute-seconds-plan.md b/docs/design/billing-compute-seconds-plan.md index 19b808f9..3e8d483d 100644 --- a/docs/design/billing-compute-seconds-plan.md +++ b/docs/design/billing-compute-seconds-plan.md @@ -431,7 +431,7 @@ plumbing exists and can be wired later. Out of scope now.) Flush UPSERT-increment of both columns + drain ship-then-delete + high-water + idempotent uuid. -### duckgres (e2e — `tests/e2e-mw-dev/harness.sh`, required per CLAUDE.md) +### duckgres (e2e — `tests/mw-dev/e2e/harness.sh`, required per CLAUDE.md) Provision an org with a known worker profile, then assert the emitted event's `cpu_seconds` ≈ cores × ceil(conn) and `memory_seconds` ≈ gib × ceil(conn): - run a query, disconnect → both metrics ≈ size × ceil(connection wall-clock). diff --git a/docs/design/resharding.md b/docs/design/resharding.md index f633a6fb..43a28472 100644 --- a/docs/design/resharding.md +++ b/docs/design/resharding.md @@ -139,7 +139,7 @@ Unit: `configstore` (tests/configstore/reshard_postgres_test.go — claim CAS, takeover fencing, cancel, log pagination, the grant-path gate), `provisioner/reshard_runner_test.go` (all three directions, rollbacks, cancel, ephemeral-password loss), `admin/reshard_test.go` (validation, secrets never -persisted). e2e (`tests/e2e-mw-dev/harness.sh`): validation 400s, +persisted). e2e (`tests/mw-dev/e2e/harness.sh`): validation 400s, cancel-during-drain (drain-not-kill + 57P03 visible), bogus-shard rollback (real flip → Synced=False → rollback, data intact), and the **ext→cnpg positive path** (real catalog move off the harness RDS onto shard-001). diff --git a/docs/design/worker-ttl-pool.md b/docs/design/worker-ttl-pool.md index d7d43d71..94d9e74b 100644 --- a/docs/design/worker-ttl-pool.md +++ b/docs/design/worker-ttl-pool.md @@ -215,4 +215,4 @@ workers cannot overcommit a node). ## Rollout Build arm64 control-plane + worker images, push to ECR, deploy to **mw-dev only** -(`tests/e2e-mw-dev/run.sh`), run harness, iterate. Never prod. +(`tests/mw-dev/run.sh`), run harness, iterate. Never prod. diff --git a/docs/perf-harness-runbook.md b/docs/perf-harness-runbook.md index 66ec945c..495770e0 100644 --- a/docs/perf-harness-runbook.md +++ b/docs/perf-harness-runbook.md @@ -5,7 +5,7 @@ Golden-query performance signal collection for `pgwire` and `flight` protocols. This is observability-only; there is no pass/fail performance gate. -Status: the old prod-us deployed perf runner path is legacy while the dev Duckgres Scenario Runner replacement is being built. Prefer `tests/scenario/scenarios/posthog_frozen_metadata.yaml` for current frozen dataset end-to-end validation; keep this perf harness for local/library use and historical artifact compatibility until the scenario perf adapter is available. +Status: the old prod-us deployed perf runner path is legacy while the dev Duckgres Scenario Runner replacement is being built. Prefer `tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml` for current frozen dataset end-to-end validation; keep this perf harness for local/library use and historical artifact compatibility until the scenario perf adapter is available. ## Local Prerequisites diff --git a/docs/runbooks/scenario-dev.md b/docs/runbooks/scenario-dev.md index 856f4164..164567e0 100644 --- a/docs/runbooks/scenario-dev.md +++ b/docs/runbooks/scenario-dev.md @@ -1,71 +1,51 @@ # Dev Scenario Runner -The `scenario-dev` GitHub Actions workflow runs scenario tests against the -configured dev environment. By default it builds the current Duckgres SHA, -deploys an isolated Duckgres stack in dev infrastructure, runs each scenario in -an in-cluster Kubernetes Job, and uploads the scenario artifacts generated by the -Go scenario harness. - -Manual runs can set `use_shared_dev` to `true` to skip the isolated deployment -and target the shared dev stack instead. Use that mode for explicit deployed-dev -checks only; the default isolated mode is the reproducible CI path. +The `scenario-dev` GitHub Actions workflow runs `scenario.full` against an +isolated Duckgres stack in mw-dev. It uses the same `tests/mw-dev` harness as +`e2e-mw-dev` for cluster access, isolated namespace deployment, pod identity, +diagnostics, and teardown. ## Scheduled Runs -The scheduled trigger runs the full daily matrix: +The scheduled trigger runs the full scenario workload group: -- `tests/scenario/scenarios/provision_smoke.yaml` -- `tests/scenario/scenarios/provision_rejection.yaml` -- `tests/scenario/scenarios/posthog_frozen_metadata.yaml` -- `tests/scenario/scenarios/posthog_frozen_perf.yaml` -- `tests/scenario/scenarios/posthog_frozen_dbt.yaml` +- provision rejection +- provision smoke +- frozen dataset metadata exploration +- frozen dataset perf queries +- frozen dataset dbt project -Manual runs default to the same full matrix. Set the `skip_slow` dispatch input -to `true` when you need a faster run that skips the slow dbt scenario. +Manual runs default to the same full group. Set the `skip_slow` dispatch input +to `true` when you want to skip the scheduled workload intentionally; the +workflow records a skipped run and does not deploy a stack. ## Required Repository Configuration The workflow follows the e2e dev harness access pattern: GitHub OIDC assumes a -dev-scoped AWS role, Tailscale reaches the private cluster API, and the scenario -runner executes inside Kubernetes. +dev-scoped AWS role, Tailscale reaches the private cluster API, and payload jobs +execute inside Kubernetes. Configure these repository secrets: - `AWS_ECR_PUBLISH_IAM_ROLE` - `MW_DEV_ACCOUNT_ID` -- `DUCKGRES_SCENARIO_CP_POD_IDENTITY_ROLE` Configure these repository variables: - `TS_WIF_CLIENT_ID_MW_DEV` - `TS_WIF_AUDIENCE_MW_DEV` -- `DUCKGRES_SCENARIO_KUBE_CONTEXT` -- `DUCKGRES_SCENARIO_EKS_CLUSTER_NAME` -- `DUCKGRES_SCENARIO_AWS_REGION` -- `DUCKGRES_SCENARIO_CONFIG_SECRET_NAMESPACE` -- `DUCKGRES_SCENARIO_CONFIG_SECRET_NAME` -- `DUCKGRES_SCENARIO_INTERNAL_SECRET_NAME` -- `DUCKGRES_SCENARIO_INTERNAL_SECRET_KEY` - -The Kubernetes scenario config secret must include `frozen-s3-uri` for frozen -dataset scenarios. Shared-dev mode additionally requires: - -- `DUCKGRES_SCENARIO_SHARED_NAMESPACE` -- `DUCKGRES_SCENARIO_SHARED_API_BASE` -- `DUCKGRES_SCENARIO_SHARED_PG_HOST` -- `DUCKGRES_SCENARIO_SHARED_FLIGHT_ADDR` -- `DUCKGRES_SCENARIO_SHARED_SNI_SUFFIX` - -Do not commit concrete dev endpoints, account ids, S3 bucket names, internal -cluster names, or secret values to this repository. Keep them in repository -secrets, repository variables, or infra-managed Kubernetes secrets. + +The workflow hardcodes the stable mw-dev cluster name/context and builds the +control-plane pod identity role ARN from `MW_DEV_ACCOUNT_ID`, matching +`e2e-mw-dev`. The scenario payload uses the isolated control-plane ClusterIP +service and the per-run internal secret generated by `tests/mw-dev/run.sh`. ## Failure Recovery -Every scenario provisions its own org or warehouse name and includes cleanup -steps. Isolated-mode runs also tear down the temporary Duckgres namespace after -the scenario Job finishes. When a run fails, inspect the uploaded -`scenario-dev-*` artifact first: +Every scenario sub-run provisions its own org or warehouse name and includes +cleanup steps. The isolated stack also tears down the temporary namespace after +the payload finishes. When a run fails, inspect the uploaded +`scenario-dev-full-*` artifact first: - `scenario_summary.json` contains the overall run verdict. - `step_results.csv` contains per-step duration, status, and error class. @@ -73,6 +53,6 @@ the scenario Job finishes. When a run fails, inspect the uploaded If cleanup did not complete, use the `run_id` in `scenario_summary.json` to find the scenario-created org name in the scenario YAML and deprovision it manually -through the relevant dev control plane. For isolated-mode failures, also inspect -the workflow logs from `tests/scenario-dev/run.sh diagnostics`; the namespace is -named from the workflow run id and matrix slot. +through the relevant dev control plane. Also inspect the workflow diagnostics +from `tests/mw-dev/run.sh diagnostics`; the namespace is named from the workflow +run id. diff --git a/docs/runbooks/scenario-runner.md b/docs/runbooks/scenario-runner.md index ad01095c..9542e01c 100644 --- a/docs/runbooks/scenario-runner.md +++ b/docs/runbooks/scenario-runner.md @@ -84,28 +84,28 @@ just scenario-frozen-dbt Run a specific scenario file: ```bash -just scenario scenario=tests/scenario/scenarios/provision_smoke.yaml +just scenario scenario=tests/mw-dev/scenario/scenarios/provision_smoke.yaml ``` Artifacts are written under `artifacts/scenario//`. The frozen metadata scenario uses: -- `tests/scenario/scenarios/posthog_frozen_metadata.yaml` -- `tests/scenario/sql/setup_frozen_views.sql` -- `tests/scenario/sql/metadata_catalog.yaml` +- `tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml` +- `tests/mw-dev/scenario/sql/setup_frozen_views.sql` +- `tests/mw-dev/scenario/sql/metadata_catalog.yaml` The frozen perf scenario uses: -- `tests/scenario/scenarios/posthog_frozen_perf.yaml` +- `tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml` - `tests/perf/queries/ducklake_frozen.yaml` Perf artifacts are written under `artifacts/scenario//perf/` using the existing `tests/perf/core` artifact schema, including `query_results.csv`, `summary.json`, and `server_metrics.prom`. The frozen dbt scenario uses: -- `tests/scenario/scenarios/posthog_frozen_dbt.yaml` -- `tests/scenario/dbt/posthog_frozen_project/` +- `tests/mw-dev/scenario/scenarios/posthog_frozen_dbt.yaml` +- `tests/mw-dev/scenario/dbt/posthog_frozen_project/` dbt artifacts are written under `artifacts/scenario//dbt/`, including per-command stdout/stderr logs, `target/` artifacts, and dbt logs. Install `dbt-postgres` locally or set `DUCKGRES_SCENARIO_DBT_BIN` to the dbt executable to use. diff --git a/justfile b/justfile index f28c587b..c0c3f540 100644 --- a/justfile +++ b/justfile @@ -300,12 +300,12 @@ test: [group('test')] test-unit: go test -v -p 1 . ./configresolve/... ./duckdbservice/... ./server/... ./transpiler/... ./internal/... ./tests/manifests/... - go test -v -count=1 ./tests/e2e-mw-dev/... + go test -v -count=1 ./tests/mw-dev/... # Run scenario runner unit tests [group('test')] test-scenario: - go test -v -count=1 ./tests/scenario/... + go test -v -count=1 ./tests/mw-dev/scenario/... # Run cache-proxy tests [group('test')] @@ -405,7 +405,7 @@ perf-nightly: # Run a Duckgres scenario file against a configured dev environment [group('test')] -scenario scenario="tests/scenario/scenarios/provision_smoke.yaml": +scenario scenario="tests/mw-dev/scenario/scenarios/provision_smoke.yaml": ./scripts/scenario_run.sh {{scenario}} # Run the dev provision smoke scenario @@ -416,27 +416,27 @@ scenario-smoke: # Run the dev provision success scenario [group('test')] scenario-provision-success: - ./scripts/scenario_run.sh tests/scenario/scenarios/provision_smoke.yaml + ./scripts/scenario_run.sh tests/mw-dev/scenario/scenarios/provision_smoke.yaml # Run the dev provision rejection scenario [group('test')] scenario-provision-rejection: - ./scripts/scenario_run.sh tests/scenario/scenarios/provision_rejection.yaml + ./scripts/scenario_run.sh tests/mw-dev/scenario/scenarios/provision_rejection.yaml # Run the dev frozen dataset metadata exploration scenario [group('test')] scenario-frozen-metadata: - ./scripts/scenario_run.sh tests/scenario/scenarios/posthog_frozen_metadata.yaml + ./scripts/scenario_run.sh tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml # Run the dev frozen dataset perf scenario [group('test')] scenario-frozen-perf: - ./scripts/scenario_run.sh tests/scenario/scenarios/posthog_frozen_perf.yaml + ./scripts/scenario_run.sh tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml # Run the dev frozen dataset dbt scenario [group('test')] scenario-frozen-dbt: - ./scripts/scenario_run.sh tests/scenario/scenarios/posthog_frozen_dbt.yaml + ./scripts/scenario_run.sh tests/mw-dev/scenario/scenarios/posthog_frozen_dbt.yaml # Lint (matches CI — uses golangci-lint, not go vet) [group('test')] diff --git a/scripts/scenario_run.sh b/scripts/scenario_run.sh index 4131770e..4ebea829 100755 --- a/scripts/scenario_run.sh +++ b/scripts/scenario_run.sh @@ -27,7 +27,7 @@ Optional environment: USAGE } -scenario_file="${DUCKGRES_SCENARIO_FILE:-tests/scenario/scenarios/provision_smoke.yaml}" +scenario_file="${DUCKGRES_SCENARIO_FILE:-tests/mw-dev/scenario/scenarios/provision_smoke.yaml}" output_base="${DUCKGRES_SCENARIO_OUTPUT_BASE:-artifacts/scenario}" run_id="${DUCKGRES_SCENARIO_RUN_ID:-}" max_runtime="${DUCKGRES_SCENARIO_MAX_RUNTIME:-30m}" @@ -129,7 +129,7 @@ if [ "$check_env_only" -eq 1 ]; then fi args=( - go test -count=1 ./tests/scenario + go test -count=1 ./tests/mw-dev/scenario -timeout "$go_test_timeout" -run TestScenarioRunner -scenario-run diff --git a/tests/manifests/manifests_test.go b/tests/manifests/manifests_test.go index 949a11f6..6895f893 100644 --- a/tests/manifests/manifests_test.go +++ b/tests/manifests/manifests_test.go @@ -2,7 +2,7 @@ // k8s/ manifests — no cluster, no build tag, runs in the normal `go test ./...` // lane. These four asserts were rescued from the retired kind suite // (tests/k8s/setup_config_test.go) when its end-to-end coverage moved to the -// real-cluster harness in tests/e2e-mw-dev/. They guard real shipped config +// real-cluster harness in tests/mw-dev/. They guard real shipped config // (k8s/rbac.yaml, k8s/networkpolicy.yaml), so they keep earning their place; // the rest of that file tested the kind-harness loader and went with it. package manifests_test diff --git a/tests/e2e-mw-dev/README.md b/tests/mw-dev/README.md similarity index 95% rename from tests/e2e-mw-dev/README.md rename to tests/mw-dev/README.md index e5213527..34f186f0 100644 --- a/tests/e2e-mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -1,27 +1,23 @@ -# mw-dev per-PR e2e harness +# mw-dev harness -Replaces what `tests/k8s/` (kind) cannot exercise — real Cilium network -policies, real Crossplane Duckling provisioning, and real cnpg-shard + -external RDS metadata stores. Those are the layers where this quarter's -production bugs lived. +Shared isolated-stack harness for dev-backed e2e and scenario tests. It owns +the real mw-dev infrastructure boundary that `tests/k8s/` (kind) cannot model: +real Cilium network policies, real Crossplane Duckling provisioning, real +cnpg-shard metadata stores, and external metadata fixtures. -## Flow (`.github/workflows/e2e-mw-dev.yml`) +## Flow 1. **Build** the arm64-only all-in-one `duckgres` image (`_image-build.yml`), - tagged `pr---arm64`, pushed to ECR. mw-dev runs ONE image for both - the control plane (`--mode control-plane`) and the workers - (`DUCKGRES_K8S_WORKER_IMAGE` = same image), so the PR builds just that. - One arch suffices (mw-dev is arm64); merge-to-main keeps the full - multi-arch + worker/CP-split matrices in the existing CD workflows. + pushed to ECR. mw-dev runs ONE image for both the control plane + (`--mode control-plane`) and the workers (`DUCKGRES_K8S_WORKER_IMAGE` = + same image), so the harness deploys one image for both roles. 2. **Tailscale** join via OIDC/WIF → reach the private mw-dev EKS API. -3. **Deploy** an isolated `duckgres-ci-pr-` namespace: throwaway - config-store Postgres + a control-plane Deployment on the PR image, spawning - worker pods in the same namespace. -4. **Test** via an in-cluster Job hitting the CP ClusterIP service. Covers - the **cnpg + ext** metadata backends. Assertions run in four **parallel - per-org lanes** (cnpg core suite, two resilience orgs, ext) — worker churn - is org-scoped, so lanes can't interfere and the wall-clock is the slowest - lane, not the sum. +3. **Deploy** an isolated `duckgres-ci-pr-` namespace: throwaway config-store + Postgres + a control-plane Deployment on the test image, spawning worker pods + in the same namespace. +4. **Test** via an in-cluster payload Job hitting the CP ClusterIP service. + `test-e2e` runs `e2e/harness.sh`; `test-scenario-full` runs the scenario + runner over the full dev workload group. 5. **Teardown** always: deprovision the ci-pr ducklings (clean shared-infra footprint) then delete the namespace. @@ -30,7 +26,7 @@ reaps any `duckgres-ci-pr-*` namespace older than 6h — a backstop for runs tha died hard before their `always()` teardown could fire. (Named e2e-cleanup, not "janitor", to avoid colliding with duckgres's own control-plane janitor.) -## What the harness asserts (`harness.sh`) +## What the e2e payload asserts (`e2e/harness.sh`) This suite is the **successor to the retired kind suite** (`tests/k8s/`): every behavior that suite asserted against a fake kind cluster is re-asserted here @@ -366,5 +362,5 @@ AWS_PROFILE=mw-dev \ NAMESPACE=duckgres-ci-pr-0 PR_NUMBER=0 KUBE_CONTEXT=posthog-mw-dev \ WORKER_IMAGE=$IMG CONTROLPLANE_IMAGE=$IMG \ CP_POD_IDENTITY_ROLE=arn:aws:iam:::role/duckgres-control-plane-dev \ - bash tests/e2e-mw-dev/run.sh deploy + bash tests/mw-dev/run.sh deploy ``` diff --git a/tests/e2e-mw-dev/harness.sh b/tests/mw-dev/e2e/harness.sh similarity index 100% rename from tests/e2e-mw-dev/harness.sh rename to tests/mw-dev/e2e/harness.sh diff --git a/tests/e2e-mw-dev/manifests.tmpl.yaml b/tests/mw-dev/manifests.tmpl.yaml similarity index 100% rename from tests/e2e-mw-dev/manifests.tmpl.yaml rename to tests/mw-dev/manifests.tmpl.yaml diff --git a/tests/e2e-mw-dev/run.sh b/tests/mw-dev/run.sh similarity index 78% rename from tests/e2e-mw-dev/run.sh rename to tests/mw-dev/run.sh index caf9620e..17dd4fa1 100755 --- a/tests/e2e-mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash -# Orchestrate the per-PR mw-dev e2e stack from the CI runner (kubectl reaches -# the private API via Tailscale). Subcommands: deploy | test | diagnostics | -# teardown. All kubectl calls pin --context explicitly — never rely on the -# current-context default. +# Orchestrate isolated mw-dev test stacks from CI (kubectl reaches the private +# API via Tailscale). Payloads are selected by subcommand; deploy/teardown and +# diagnostics are shared by e2e and scenario runs. # -# Required env (set by .github/workflows/e2e-mw-dev.yml): +# Required env (set by .github/workflows/*mw-dev.yml): # NAMESPACE, PR_NUMBER, WORKER_IMAGE, CONTROLPLANE_IMAGE, KUBE_CONTEXT, # CP_POD_IDENTITY_ROLE (the duckgres-control-plane-dev role ARN — the per-PR # CP assumes the SAME EKS Pod Identity as the real mw-dev control plane, so @@ -21,6 +20,9 @@ KUBECTL=(kubectl --context "$CTX") EKS_CLUSTER_NAME="${EKS_CLUSTER_NAME:-posthog-mw-dev}" AWS_REGION="${AWS_REGION:-us-east-1}" SA_NAME="duckgres" +FROZEN_S3_URI="${DUCKGRES_SCENARIO_FROZEN_S3_URI:-s3://posthog-duckgres-scenario-frozen-data-mw-dev/frozen_v1/}" +SCENARIO_JOB_WATCH_TIMEOUT_SECONDS="${SCENARIO_JOB_WATCH_TIMEOUT_SECONDS:-16200}" +SCENARIO_FULL_FILES="${SCENARIO_FULL_FILES:-tests/mw-dev/scenario/scenarios/provision_rejection.yaml tests/mw-dev/scenario/scenarios/provision_smoke.yaml tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml tests/mw-dev/scenario/scenarios/posthog_frozen_dbt.yaml}" # Internal secret for the per-PR control plane. Random per run; never reused. # Stamped into the rendered manifests and handed to the in-cluster harness. @@ -222,12 +224,12 @@ cmd_deploy() { restart_cp_with_identity } -cmd_test() { +cmd_test_e2e() { # Ship harness.sh into the namespace as a ConfigMap and run it as a Job that # talks to the control-plane ClusterIP service. The Job SA is `duckgres`, # which can delete worker pods in-namespace (durability test). "${KUBECTL[@]}" -n "$NS" create configmap duckgres-harness \ - --from-file=harness.sh="$HERE/harness.sh" \ + --from-file=harness.sh="$HERE/e2e/harness.sh" \ --dry-run=client -o yaml | "${KUBECTL[@]}" apply -f - INTERNAL_SECRET="$(cat "$internal_secret_file")" @@ -304,6 +306,117 @@ YAML return 1 } +scenario_name_for_file() { + basename "$1" .yaml | tr '_' '-' +} + +scenario_job_name() { + local name="$1" run_hash + run_hash="$(printf '%s' "${DUCKGRES_SCENARIO_RUN_ID:?DUCKGRES_SCENARIO_RUN_ID is required}" | cksum | awk '{print $1}')" + printf 'duckgres-scenario-%s-%s\n' "$name" "$run_hash" | tr '_' '-' +} + +cmd_test_scenario_full() { + local scenario_file scenario_name scenario_rc rc=0 + : "${SCENARIO_RUNNER_IMAGE:?SCENARIO_RUNNER_IMAGE is required}" + + for scenario_file in $SCENARIO_FULL_FILES; do + scenario_name="$(scenario_name_for_file "$scenario_file")" + DUCKGRES_SCENARIO_RUN_ID="scenario-dev-${scenario_name}-${PR_NUMBER}" \ + run_scenario "$scenario_name" "$scenario_file" || { + scenario_rc=$? + [ "$rc" -ne 0 ] || rc="$scenario_rc" + } + done + return "$rc" +} + +run_scenario() { + local scenario_name="$1" scenario_file="$2" job api_base pg flight suffix internal_secret rc=0 + api_base="http://duckgres-control-plane.$NS.svc:8080" + pg="$("${KUBECTL[@]}" -n "$NS" get svc duckgres-control-plane -o jsonpath='{.spec.clusterIP}')" + flight="duckgres-control-plane.$NS.svc:8815" + suffix=".ci.duckgres.local" + internal_secret="$(cat "$internal_secret_file")" + job="$(scenario_job_name "$scenario_name")" + + "${KUBECTL[@]}" -n "$NS" delete job "$job" --ignore-not-found + cat </dev/null)" = "True" ]; then + echo "scenario Job $job complete." + return 0 + fi + if [ "$("${KUBECTL[@]}" -n "$NS" get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null)" = "True" ]; then + echo "scenario Job $job failed." >&2 + return 1 + fi + sleep 10 + done + echo "scenario Job $job did not reach a terminal state in time." >&2 + return 1 +} + +copy_scenario_artifacts() { + local scenario_name="$1" job="$2" pod dest + pod="$("${KUBECTL[@]}" -n "$NS" get pod -l "job-name=$job" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + [ -n "$pod" ] || return 0 + dest="$HERE/../../artifacts/scenario-dev/${scenario_name}" + mkdir -p "$dest" + "${KUBECTL[@]}" -n "$NS" cp "$pod:/artifacts/scenario-dev" "$dest" >/dev/null 2>&1 || true +} + cmd_diagnostics() { echo "::group::namespace state" "${KUBECTL[@]}" -n "$NS" get pods,svc,job -o wide || true @@ -319,6 +432,9 @@ cmd_diagnostics() { echo "::group::worker pods" "${KUBECTL[@]}" -n "$NS" get pods -l app=duckgres-worker -o wide || true echo "::endgroup::" + echo "::group::scenario jobs" + "${KUBECTL[@]}" -n "$NS" get job,pod -l job-name -o wide || true + echo "::endgroup::" } cmd_teardown() { @@ -422,9 +538,10 @@ cmd_e2e_cleanup() { done } -case "${1:?usage: run.sh deploy|test|diagnostics|teardown|e2e-cleanup}" in +case "${1:?usage: run.sh deploy|test-e2e|test-scenario-full|diagnostics|teardown|e2e-cleanup}" in deploy) : "${NAMESPACE:?}"; require_pr_identity; cmd_deploy ;; - test) : "${NAMESPACE:?}"; require_pr_identity; cmd_test ;; + test-e2e|test) : "${NAMESPACE:?}"; require_pr_identity; cmd_test_e2e ;; + test-scenario-full) : "${NAMESPACE:?}"; require_pr_identity; cmd_test_scenario_full ;; diagnostics) : "${NAMESPACE:?}"; require_pr_identity; cmd_diagnostics ;; teardown) : "${NAMESPACE:?}"; require_pr_identity; cmd_teardown ;; e2e-cleanup) cmd_e2e_cleanup ;; diff --git a/tests/e2e-mw-dev/run_sh_test.go b/tests/mw-dev/run_sh_test.go similarity index 67% rename from tests/e2e-mw-dev/run_sh_test.go rename to tests/mw-dev/run_sh_test.go index eca0e5d3..78793766 100644 --- a/tests/e2e-mw-dev/run_sh_test.go +++ b/tests/mw-dev/run_sh_test.go @@ -118,6 +118,88 @@ func TestDeployRejectsMissingPRNumberBeforeCleanup(t *testing.T) { } } +func TestScenarioFullRunsScenarioJobsAgainstIsolatedStack(t *testing.T) { + fakes := newRunSHFakes(t) + + cmd := runSHCommand(t, fakes.binDir, "test-scenario-full", + "SCENARIO_RUNNER_IMAGE=example.invalid/duckgres:scenario", + "SCENARIO_FULL_FILES=tests/mw-dev/scenario/scenarios/provision_smoke.yaml tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml", + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("scenario full failed: %v\n%s", err, out) + } + + calls := fakes.calls(t) + for _, want := range []string{ + "kubectl --context test-context -n duckgres-ci-pr-123 get svc duckgres-control-plane -o jsonpath={.spec.clusterIP}", + "kubectl --context test-context -n duckgres-ci-pr-123 apply -f -", + "kubectl --context test-context -n duckgres-ci-pr-123 logs -f job/duckgres-scenario-provision-smoke-", + "kubectl --context test-context -n duckgres-ci-pr-123 logs -f job/duckgres-scenario-posthog-frozen-metadata-", + } { + if !strings.Contains(calls, want) { + t.Fatalf("scenario full missing expected call %q; calls:\n%s", want, calls) + } + } + if !strings.Contains(calls, "s3://posthog-duckgres-scenario-frozen-data-mw-dev/frozen_v1/") { + t.Fatalf("scenario full did not pass the frozen dataset URI in the Job manifest; calls:\n%s", calls) + } + if strings.Contains(calls, "get secret duckgres-scenario-config") { + t.Fatalf("scenario full still reads a scenario config secret; calls:\n%s", calls) + } +} + +func TestScenarioFullContinuesAfterScenarioFailure(t *testing.T) { + fakes := newRunSHFakes(t) + + cmd := runSHCommand(t, fakes.binDir, "test-scenario-full", + "SCENARIO_RUNNER_IMAGE=example.invalid/duckgres:scenario", + "SCENARIO_FULL_FILES=tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml", + "SCENARIO_DEV_FAIL_JOB=posthog-frozen-metadata", + ) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("scenario full succeeded despite a failed subscenario; output:\n%s", out) + } + + calls := fakes.calls(t) + for _, want := range []string{ + "kubectl --context test-context -n duckgres-ci-pr-123 logs -f job/duckgres-scenario-posthog-frozen-metadata-", + "kubectl --context test-context -n duckgres-ci-pr-123 logs -f job/duckgres-scenario-posthog-frozen-perf-", + } { + if !strings.Contains(calls, want) { + t.Fatalf("scenario full did not continue through expected call %q; calls:\n%s", want, calls) + } + } +} + +func TestRunScriptUsesMwDevPayloadLayout(t *testing.T) { + raw, err := os.ReadFile("run.sh") + if err != nil { + t.Fatalf("read run.sh: %v", err) + } + script := string(raw) + for _, want := range []string{ + "$HERE/e2e/harness.sh", + "test-scenario-full", + "SCENARIO_FULL_FILES", + ".ci.duckgres.local", + } { + if !strings.Contains(script, want) { + t.Fatalf("run.sh missing %q", want) + } + } + for _, forbidden := range []string{ + "USE_SHARED_DEV", + "SCENARIO_SHARED_", + "SCENARIO_CONFIG_SECRET", + } { + if strings.Contains(script, forbidden) { + t.Fatalf("run.sh still contains shared-dev/config-secret path %q", forbidden) + } + } +} + type runSHFakes struct { binDir string logPath string @@ -137,7 +219,28 @@ func newRunSHFakes(t *testing.T) runSHFakes { printf 'kubectl %s\n' "$*" >> "$RUN_SH_TEST_CALLS" if [[ "$*" == *" apply -f -"* ]]; then - cat >/dev/null + tee -a "$RUN_SH_TEST_CALLS" >/dev/null + exit 0 +fi +if [[ "$*" == *" get svc duckgres-control-plane "* ]]; then + printf '10.96.0.20' + exit 0 +fi +if [[ "$*" == *" get job duckgres-scenario-"* ]]; then + if [[ -n "${SCENARIO_DEV_FAIL_JOB:-}" && "$*" == *"duckgres-scenario-${SCENARIO_DEV_FAIL_JOB}-"* ]]; then + if [[ "$*" == *'@.type=="Failed"'* ]]; then + printf 'True' + fi + exit 0 + fi + if [[ "$*" == *'@.type=="Failed"'* ]]; then + exit 0 + fi + printf 'True' + exit 0 +fi +if [[ "$*" == *" get pod -l job-name=duckgres-scenario-"* ]]; then + printf 'duckgres-scenario-pod' exit 0 fi if [[ "$*" == *" wait --for=delete duckling/ci-pr-123-"* ]]; then diff --git a/tests/scenario/Dockerfile b/tests/mw-dev/scenario/Dockerfile similarity index 90% rename from tests/scenario/Dockerfile rename to tests/mw-dev/scenario/Dockerfile index d343946b..8573ea93 100644 --- a/tests/scenario/Dockerfile +++ b/tests/mw-dev/scenario/Dockerfile @@ -11,7 +11,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go test -run '^$' ./tests/scenario +RUN go test -run '^$' ./tests/mw-dev/scenario ENV DUCKGRES_SCENARIO_DBT_BIN=dbt diff --git a/tests/scenario/core/artifacts.go b/tests/mw-dev/scenario/core/artifacts.go similarity index 100% rename from tests/scenario/core/artifacts.go rename to tests/mw-dev/scenario/core/artifacts.go diff --git a/tests/scenario/core/artifacts_test.go b/tests/mw-dev/scenario/core/artifacts_test.go similarity index 100% rename from tests/scenario/core/artifacts_test.go rename to tests/mw-dev/scenario/core/artifacts_test.go diff --git a/tests/scenario/core/catalog.go b/tests/mw-dev/scenario/core/catalog.go similarity index 100% rename from tests/scenario/core/catalog.go rename to tests/mw-dev/scenario/core/catalog.go diff --git a/tests/scenario/core/catalog_test.go b/tests/mw-dev/scenario/core/catalog_test.go similarity index 100% rename from tests/scenario/core/catalog_test.go rename to tests/mw-dev/scenario/core/catalog_test.go diff --git a/tests/scenario/core/runner.go b/tests/mw-dev/scenario/core/runner.go similarity index 100% rename from tests/scenario/core/runner.go rename to tests/mw-dev/scenario/core/runner.go diff --git a/tests/scenario/core/runner_test.go b/tests/mw-dev/scenario/core/runner_test.go similarity index 100% rename from tests/scenario/core/runner_test.go rename to tests/mw-dev/scenario/core/runner_test.go diff --git a/tests/scenario/core/templates.go b/tests/mw-dev/scenario/core/templates.go similarity index 100% rename from tests/scenario/core/templates.go rename to tests/mw-dev/scenario/core/templates.go diff --git a/tests/scenario/dbt/errors.go b/tests/mw-dev/scenario/dbt/errors.go similarity index 100% rename from tests/scenario/dbt/errors.go rename to tests/mw-dev/scenario/dbt/errors.go diff --git a/tests/scenario/dbt/posthog_frozen_project/dbt_project.yml b/tests/mw-dev/scenario/dbt/posthog_frozen_project/dbt_project.yml similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/dbt_project.yml rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/dbt_project.yml diff --git a/tests/scenario/dbt/posthog_frozen_project/models/facts/fct_activation_funnel.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_activation_funnel.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/facts/fct_activation_funnel.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_activation_funnel.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/facts/fct_feature_usage_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_feature_usage_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/facts/fct_feature_usage_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_feature_usage_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/facts/fct_retention_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_retention_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/facts/fct_retention_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_retention_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/facts/fct_sessions.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_sessions.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/facts/fct_sessions.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_sessions.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/facts/fct_user_activity_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_user_activity_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/facts/fct_user_activity_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/facts/fct_user_activity_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_event_days.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_event_days.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_event_days.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_event_days.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_activity_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_activity_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_activity_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_activity_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_feature_usage_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_feature_usage_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_feature_usage_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_feature_usage_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_first_seen.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_first_seen.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_first_seen.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_person_first_seen.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_sessionized_events.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_sessionized_events.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/intermediate/int_sessionized_events.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/intermediate/int_sessionized_events.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/marts/mart_product_kpis_daily.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/marts/mart_product_kpis_daily.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/marts/mart_product_kpis_daily.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/marts/mart_product_kpis_daily.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/schema.yml b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/schema.yml similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/schema.yml rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/schema.yml diff --git a/tests/scenario/dbt/posthog_frozen_project/models/sources.yml b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/sources.yml similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/sources.yml rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/sources.yml diff --git a/tests/scenario/dbt/posthog_frozen_project/models/staging/stg_events.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/staging/stg_events.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/staging/stg_events.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/staging/stg_events.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/models/staging/stg_persons.sql b/tests/mw-dev/scenario/dbt/posthog_frozen_project/models/staging/stg_persons.sql similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/models/staging/stg_persons.sql rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/models/staging/stg_persons.sql diff --git a/tests/scenario/dbt/posthog_frozen_project/profiles.yml b/tests/mw-dev/scenario/dbt/posthog_frozen_project/profiles.yml similarity index 100% rename from tests/scenario/dbt/posthog_frozen_project/profiles.yml rename to tests/mw-dev/scenario/dbt/posthog_frozen_project/profiles.yml diff --git a/tests/scenario/dbt/runner_test.go b/tests/mw-dev/scenario/dbt/runner_test.go similarity index 98% rename from tests/scenario/dbt/runner_test.go rename to tests/mw-dev/scenario/dbt/runner_test.go index 2bae7a09..07f11c72 100644 --- a/tests/scenario/dbt/runner_test.go +++ b/tests/mw-dev/scenario/dbt/runner_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" - scenariosql "github.com/posthog/duckgres/tests/scenario/sql" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" + scenariosql "github.com/posthog/duckgres/tests/mw-dev/scenario/sql" ) func TestExecutorRunsCommandsCapturesLogsAndCopiesTargetArtifacts(t *testing.T) { diff --git a/tests/scenario/dbt/steps.go b/tests/mw-dev/scenario/dbt/steps.go similarity index 98% rename from tests/scenario/dbt/steps.go rename to tests/mw-dev/scenario/dbt/steps.go index a4d23b78..9779bd6b 100644 --- a/tests/scenario/dbt/steps.go +++ b/tests/mw-dev/scenario/dbt/steps.go @@ -12,9 +12,9 @@ import ( "strings" "sync" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" - scenariosql "github.com/posthog/duckgres/tests/scenario/sql" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" + scenariosql "github.com/posthog/duckgres/tests/mw-dev/scenario/sql" ) const StepTypeDBTRun = "dbt_run" diff --git a/tests/scenario/perf/adapter_test.go b/tests/mw-dev/scenario/perf/adapter_test.go similarity index 98% rename from tests/scenario/perf/adapter_test.go rename to tests/mw-dev/scenario/perf/adapter_test.go index aa1e48d1..70d5730f 100644 --- a/tests/scenario/perf/adapter_test.go +++ b/tests/mw-dev/scenario/perf/adapter_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" + scenariosql "github.com/posthog/duckgres/tests/mw-dev/scenario/sql" perfcore "github.com/posthog/duckgres/tests/perf/core" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" - scenariosql "github.com/posthog/duckgres/tests/scenario/sql" ) func TestExecutorRunsPerfStepAndWritesArtifacts(t *testing.T) { diff --git a/tests/scenario/perf/errors.go b/tests/mw-dev/scenario/perf/errors.go similarity index 100% rename from tests/scenario/perf/errors.go rename to tests/mw-dev/scenario/perf/errors.go diff --git a/tests/scenario/perf/steps.go b/tests/mw-dev/scenario/perf/steps.go similarity index 98% rename from tests/scenario/perf/steps.go rename to tests/mw-dev/scenario/perf/steps.go index 63f6bcf4..1785a025 100644 --- a/tests/scenario/perf/steps.go +++ b/tests/mw-dev/scenario/perf/steps.go @@ -8,12 +8,12 @@ import ( "sync" "time" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" + scenariosql "github.com/posthog/duckgres/tests/mw-dev/scenario/sql" perfcore "github.com/posthog/duckgres/tests/perf/core" flightdriver "github.com/posthog/duckgres/tests/perf/drivers/flight" pgdriver "github.com/posthog/duckgres/tests/perf/drivers/pgwire" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" - scenariosql "github.com/posthog/duckgres/tests/scenario/sql" ) const StepTypePerfQueries = "perf_queries" diff --git a/tests/scenario/provision/client.go b/tests/mw-dev/scenario/provision/client.go similarity index 100% rename from tests/scenario/provision/client.go rename to tests/mw-dev/scenario/provision/client.go diff --git a/tests/scenario/provision/client_test.go b/tests/mw-dev/scenario/provision/client_test.go similarity index 100% rename from tests/scenario/provision/client_test.go rename to tests/mw-dev/scenario/provision/client_test.go diff --git a/tests/scenario/provision/errors.go b/tests/mw-dev/scenario/provision/errors.go similarity index 100% rename from tests/scenario/provision/errors.go rename to tests/mw-dev/scenario/provision/errors.go diff --git a/tests/scenario/provision/redact.go b/tests/mw-dev/scenario/provision/redact.go similarity index 100% rename from tests/scenario/provision/redact.go rename to tests/mw-dev/scenario/provision/redact.go diff --git a/tests/scenario/provision/steps.go b/tests/mw-dev/scenario/provision/steps.go similarity index 99% rename from tests/scenario/provision/steps.go rename to tests/mw-dev/scenario/provision/steps.go index 13b8d296..51c5634b 100644 --- a/tests/scenario/provision/steps.go +++ b/tests/mw-dev/scenario/provision/steps.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/posthog/duckgres/tests/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" ) const ( diff --git a/tests/scenario/provision/steps_test.go b/tests/mw-dev/scenario/provision/steps_test.go similarity index 99% rename from tests/scenario/provision/steps_test.go rename to tests/mw-dev/scenario/provision/steps_test.go index b7c2d0cc..9b779e1e 100644 --- a/tests/scenario/provision/steps_test.go +++ b/tests/mw-dev/scenario/provision/steps_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/posthog/duckgres/tests/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" ) func TestExecutorProvisionStepStoresPasswordForLaterSteps(t *testing.T) { diff --git a/tests/scenario/runner_test.go b/tests/mw-dev/scenario/runner_test.go similarity index 98% rename from tests/scenario/runner_test.go rename to tests/mw-dev/scenario/runner_test.go index f14ab56e..7d4d0e9e 100644 --- a/tests/scenario/runner_test.go +++ b/tests/mw-dev/scenario/runner_test.go @@ -12,11 +12,11 @@ import ( "testing" "time" - "github.com/posthog/duckgres/tests/scenario/core" - scenariodbt "github.com/posthog/duckgres/tests/scenario/dbt" - scenarioperf "github.com/posthog/duckgres/tests/scenario/perf" - "github.com/posthog/duckgres/tests/scenario/provision" - scenariosql "github.com/posthog/duckgres/tests/scenario/sql" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + scenariodbt "github.com/posthog/duckgres/tests/mw-dev/scenario/dbt" + scenarioperf "github.com/posthog/duckgres/tests/mw-dev/scenario/perf" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" + scenariosql "github.com/posthog/duckgres/tests/mw-dev/scenario/sql" ) var ( diff --git a/tests/scenario/scenarios/posthog_frozen_dbt.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_dbt.yaml similarity index 100% rename from tests/scenario/scenarios/posthog_frozen_dbt.yaml rename to tests/mw-dev/scenario/scenarios/posthog_frozen_dbt.yaml diff --git a/tests/scenario/scenarios/posthog_frozen_metadata.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml similarity index 100% rename from tests/scenario/scenarios/posthog_frozen_metadata.yaml rename to tests/mw-dev/scenario/scenarios/posthog_frozen_metadata.yaml diff --git a/tests/scenario/scenarios/posthog_frozen_perf.yaml b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml similarity index 95% rename from tests/scenario/scenarios/posthog_frozen_perf.yaml rename to tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml index d5493d92..4891939c 100644 --- a/tests/scenario/scenarios/posthog_frozen_perf.yaml +++ b/tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml @@ -39,7 +39,7 @@ steps: with: org_id: scenario-frozen-perf-${run_id_token} catalog: ducklake - catalog_file: ../../perf/queries/ducklake_frozen.yaml + catalog_file: ../../../perf/queries/ducklake_frozen.yaml run_id: ${run_id} dataset_version: posthog-file-views-v1 flight_addr: ${env:DUCKGRES_SCENARIO_FLIGHT_ADDR} diff --git a/tests/scenario/scenarios/provision_rejection.yaml b/tests/mw-dev/scenario/scenarios/provision_rejection.yaml similarity index 100% rename from tests/scenario/scenarios/provision_rejection.yaml rename to tests/mw-dev/scenario/scenarios/provision_rejection.yaml diff --git a/tests/scenario/scenarios/provision_smoke.yaml b/tests/mw-dev/scenario/scenarios/provision_smoke.yaml similarity index 100% rename from tests/scenario/scenarios/provision_smoke.yaml rename to tests/mw-dev/scenario/scenarios/provision_smoke.yaml diff --git a/tests/scenario/script_test.go b/tests/mw-dev/scenario/script_test.go similarity index 69% rename from tests/scenario/script_test.go rename to tests/mw-dev/scenario/script_test.go index bf03e86d..e05d9365 100644 --- a/tests/scenario/script_test.go +++ b/tests/mw-dev/scenario/script_test.go @@ -9,7 +9,7 @@ import ( ) func TestScenarioRunScriptValidatesRequiredEnvVars(t *testing.T) { - script := filepath.Join("..", "..", "scripts", "scenario_run.sh") + script := filepath.Join("..", "..", "..", "scripts", "scenario_run.sh") cmd := exec.Command("bash", script, "--check-env") cmd.Env = []string{"PATH=" + os.Getenv("PATH")} @@ -31,8 +31,8 @@ func TestScenarioRunScriptValidatesRequiredEnvVars(t *testing.T) { } func TestScenarioRunScriptCheckEnvIncludesScenarioRequiredEnv(t *testing.T) { - script := filepath.Join("..", "..", "scripts", "scenario_run.sh") - cmd := exec.Command("bash", script, "--check-env", "tests/scenario/scenarios/posthog_frozen_perf.yaml") + script := filepath.Join("..", "..", "..", "scripts", "scenario_run.sh") + cmd := exec.Command("bash", script, "--check-env", "tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml") cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), "DUCKGRES_SCENARIO_API_BASE=http://127.0.0.1", @@ -56,8 +56,8 @@ func TestScenarioRunScriptCheckEnvIncludesScenarioRequiredEnv(t *testing.T) { } } -func TestDevScenarioWorkflowDefaultsToIsolatedDeploymentWithSharedDevOverride(t *testing.T) { - workflowPath := filepath.Join("..", "..", ".github", "workflows", "scenario-dev.yml") +func TestDevScenarioWorkflowUsesUnifiedMwDevHarness(t *testing.T) { + workflowPath := filepath.Join("..", "..", "..", ".github", "workflows", "scenario-dev.yml") raw, err := os.ReadFile(workflowPath) if err != nil { t.Fatalf("read dev scenario workflow: %v", err) @@ -68,7 +68,6 @@ func TestDevScenarioWorkflowDefaultsToIsolatedDeploymentWithSharedDevOverride(t "name: scenario-dev", "workflow_dispatch:", "skip_slow:", - "use_shared_dev:", "default: false", "schedule:", "id-token: write", @@ -77,20 +76,15 @@ func TestDevScenarioWorkflowDefaultsToIsolatedDeploymentWithSharedDevOverride(t "tag: scenario-runner-${{ github.run_id }}-${{ github.run_attempt }}-arm64", "tag: scenario-duckgres-${{ github.run_id }}-${{ github.run_attempt }}-arm64", "inputs.duckgres_image != ''", - "tests/scenario-dev/run.sh deploy", - "tests/scenario-dev/run.sh test", - "tests/scenario-dev/run.sh diagnostics", - "tests/scenario-dev/run.sh teardown", - "tests/scenario/scenarios/provision_smoke.yaml", - "tests/scenario/scenarios/provision_rejection.yaml", - "tests/scenario/scenarios/posthog_frozen_metadata.yaml", - "tests/scenario/scenarios/posthog_frozen_perf.yaml", - "tests/scenario/scenarios/posthog_frozen_dbt.yaml", - "if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_slow || !matrix.slow }}", - "if: ${{ github.event_name != 'workflow_dispatch' || !inputs.use_shared_dev }}", - "if: ${{ github.event_name == 'workflow_dispatch' && inputs.use_shared_dev }}", - "slow: true", - "go test -count=1 ./tests/scenario ./tests/scenario-dev ./tests/e2e-mw-dev", + "KUBE_CONTEXT: posthog-mw-dev", + "CLUSTER_NAME: posthog-mw-dev", + "EKS_CLUSTER_NAME: posthog-mw-dev", + "CP_POD_IDENTITY_ROLE: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/duckgres-control-plane-dev", + "tests/mw-dev/run.sh deploy", + "tests/mw-dev/run.sh test-scenario-full", + "tests/mw-dev/run.sh diagnostics", + "tests/mw-dev/run.sh teardown", + "go test -count=1 ./tests/mw-dev/scenario ./tests/mw-dev", } { if !strings.Contains(workflow, required) { t.Fatalf("workflow missing %q", required) @@ -98,16 +92,24 @@ func TestDevScenarioWorkflowDefaultsToIsolatedDeploymentWithSharedDevOverride(t } for _, forbidden := range []string{ + "use_shared_dev:", + "USE_SHARED_DEV", + "SCENARIO_SHARED_", + "DUCKGRES_SCENARIO_KUBE_CONTEXT", + "DUCKGRES_SCENARIO_EKS_CLUSTER_NAME", + "DUCKGRES_SCENARIO_CP_POD_IDENTITY_ROLE", + "DUCKGRES_SCENARIO_CONFIG_SECRET", + "DUCKGRES_SCENARIO_INTERNAL_SECRET_NAME", + "DUCKGRES_SCENARIO_INTERNAL_SECRET_KEY", + "matrix:", "DUCKGRES_SCENARIO_API_BASE: ${{ secrets.", "DUCKGRES_SCENARIO_INTERNAL_SECRET: ${{ secrets.", "DUCKGRES_SCENARIO_PG_HOST: ${{ secrets.", "DUCKGRES_SCENARIO_SNI_SUFFIX: ${{ secrets.", "DUCKGRES_SCENARIO_FROZEN_S3_URI: ${{ secrets.", "DUCKGRES_SCENARIO_FLIGHT_ADDR: ${{ secrets.", - "posthog-mw-dev", "373313242555", "645773004826", - "posthog-duckgres-scenario-frozen-data", "posthog-duckling-perfprodus", } { if strings.Contains(workflow, forbidden) { @@ -126,7 +128,7 @@ func TestScenarioRunnerImageCachesGoDependencies(t *testing.T) { for _, required := range []string{ "go mod download", - "go test -run '^$' ./tests/scenario", + "go test -run '^$' ./tests/mw-dev/scenario", } { if !strings.Contains(dockerfile, required) { t.Fatalf("scenario runner Dockerfile missing %q", required) diff --git a/tests/scenario/sql/connection.go b/tests/mw-dev/scenario/sql/connection.go similarity index 100% rename from tests/scenario/sql/connection.go rename to tests/mw-dev/scenario/sql/connection.go diff --git a/tests/scenario/sql/connection_test.go b/tests/mw-dev/scenario/sql/connection_test.go similarity index 100% rename from tests/scenario/sql/connection_test.go rename to tests/mw-dev/scenario/sql/connection_test.go diff --git a/tests/scenario/sql/driver.go b/tests/mw-dev/scenario/sql/driver.go similarity index 100% rename from tests/scenario/sql/driver.go rename to tests/mw-dev/scenario/sql/driver.go diff --git a/tests/scenario/sql/errors.go b/tests/mw-dev/scenario/sql/errors.go similarity index 100% rename from tests/scenario/sql/errors.go rename to tests/mw-dev/scenario/sql/errors.go diff --git a/tests/scenario/sql/metadata_catalog.yaml b/tests/mw-dev/scenario/sql/metadata_catalog.yaml similarity index 100% rename from tests/scenario/sql/metadata_catalog.yaml rename to tests/mw-dev/scenario/sql/metadata_catalog.yaml diff --git a/tests/scenario/sql/setup_frozen_views.sql b/tests/mw-dev/scenario/sql/setup_frozen_views.sql similarity index 100% rename from tests/scenario/sql/setup_frozen_views.sql rename to tests/mw-dev/scenario/sql/setup_frozen_views.sql diff --git a/tests/scenario/sql/steps.go b/tests/mw-dev/scenario/sql/steps.go similarity index 99% rename from tests/scenario/sql/steps.go rename to tests/mw-dev/scenario/sql/steps.go index 19388a52..5ffe2b9a 100644 --- a/tests/scenario/sql/steps.go +++ b/tests/mw-dev/scenario/sql/steps.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" "gopkg.in/yaml.v3" ) diff --git a/tests/scenario/sql/steps_test.go b/tests/mw-dev/scenario/sql/steps_test.go similarity index 99% rename from tests/scenario/sql/steps_test.go rename to tests/mw-dev/scenario/sql/steps_test.go index 158be219..785f6c70 100644 --- a/tests/scenario/sql/steps_test.go +++ b/tests/mw-dev/scenario/sql/steps_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/posthog/duckgres/tests/scenario/core" - "github.com/posthog/duckgres/tests/scenario/provision" + "github.com/posthog/duckgres/tests/mw-dev/scenario/core" + "github.com/posthog/duckgres/tests/mw-dev/scenario/provision" ) func TestExecutorRetriesTransientStartupErrors(t *testing.T) { diff --git a/tests/scenario-dev/run.sh b/tests/scenario-dev/run.sh deleted file mode 100755 index d8dc63ff..00000000 --- a/tests/scenario-dev/run.sh +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env bash -# Run scenario-dev against either a per-run isolated Duckgres stack or the -# shared dev stack. The isolated stack is delegated to tests/e2e-mw-dev/run.sh; -# this script owns only the scenario-runner Kubernetes Job. -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -CTX="${KUBE_CONTEXT:?KUBE_CONTEXT is required}" -KUBECTL=(kubectl --context "$CTX") -USE_SHARED_DEV="${USE_SHARED_DEV:-false}" -SCENARIO_NAMESPACE="${SCENARIO_NAMESPACE:-${NAMESPACE:-}}" -SCENARIO_SHARED_NAMESPACE="${SCENARIO_SHARED_NAMESPACE:-duckgres}" -SCENARIO_CONFIG_SECRET_NAME="${SCENARIO_CONFIG_SECRET_NAME:-duckgres-scenario-config}" -SCENARIO_CONFIG_SECRET_NAMESPACE="${SCENARIO_CONFIG_SECRET_NAMESPACE:-$SCENARIO_SHARED_NAMESPACE}" -SCENARIO_INTERNAL_SECRET_NAME="${SCENARIO_INTERNAL_SECRET_NAME:-duckgres-tokens}" -SCENARIO_INTERNAL_SECRET_KEY="${SCENARIO_INTERNAL_SECRET_KEY:-internal-secret}" -SCENARIO_ISOLATED_SNI_SUFFIX="${SCENARIO_ISOLATED_SNI_SUFFIX:-.ci.duckgres.local}" -SCENARIO_SHARED_SNI_SUFFIX="${SCENARIO_SHARED_SNI_SUFFIX:-${SCENARIO_SNI_SUFFIX:-}}" -SCENARIO_JOB_WATCH_TIMEOUT_SECONDS="${SCENARIO_JOB_WATCH_TIMEOUT_SECONDS:-16200}" - -target_namespace() { - if [ "$USE_SHARED_DEV" = "true" ]; then - printf '%s\n' "$SCENARIO_SHARED_NAMESPACE" - else - : "${SCENARIO_NAMESPACE:?SCENARIO_NAMESPACE or NAMESPACE is required}" - printf '%s\n' "$SCENARIO_NAMESPACE" - fi -} - -control_plane_api_base() { - local ns - ns="$(target_namespace)" - if [ "$USE_SHARED_DEV" = "true" ]; then - printf '%s\n' "${SCENARIO_SHARED_API_BASE:?SCENARIO_SHARED_API_BASE is required when USE_SHARED_DEV=true}" - else - printf 'http://duckgres-control-plane.%s.svc:8080\n' "$ns" - fi -} - -pg_host() { - local ns - ns="$(target_namespace)" - if [ "$USE_SHARED_DEV" = "true" ]; then - printf '%s\n' "${SCENARIO_SHARED_PG_HOST:?SCENARIO_SHARED_PG_HOST is required when USE_SHARED_DEV=true}" - else - "${KUBECTL[@]}" -n "$ns" get svc duckgres-control-plane -o jsonpath='{.spec.clusterIP}' - fi -} - -flight_addr() { - local ns - ns="$(target_namespace)" - if [ "$USE_SHARED_DEV" = "true" ]; then - printf '%s\n' "${SCENARIO_SHARED_FLIGHT_ADDR:?SCENARIO_SHARED_FLIGHT_ADDR is required when USE_SHARED_DEV=true}" - else - printf 'duckgres-control-plane.%s.svc:8815\n' "$ns" - fi -} - -sni_suffix() { - if [ "$USE_SHARED_DEV" = "true" ]; then - printf '%s\n' "${SCENARIO_SHARED_SNI_SUFFIX:?SCENARIO_SHARED_SNI_SUFFIX is required when USE_SHARED_DEV=true}" - else - printf '%s\n' "$SCENARIO_ISOLATED_SNI_SUFFIX" - fi -} - -ensure_config_secret() { - local target_ns source_ns name - target_ns="$(target_namespace)" - source_ns="$SCENARIO_CONFIG_SECRET_NAMESPACE" - name="$SCENARIO_CONFIG_SECRET_NAME" - if [ "$target_ns" = "$source_ns" ]; then - return 0 - fi - "${KUBECTL[@]}" -n "$source_ns" get secret "$name" -o json \ - | jq --arg ns "$target_ns" 'del( - .metadata.annotations, - .metadata.creationTimestamp, - .metadata.managedFields, - .metadata.resourceVersion, - .metadata.uid - ) | .metadata.namespace = $ns' \ - | "${KUBECTL[@]}" apply -f - -} - -scenario_requires_frozen_config() { - local scenario_path - case "$SCENARIO_FILE" in - /*) scenario_path="$SCENARIO_FILE" ;; - *) scenario_path="$ROOT/$SCENARIO_FILE" ;; - esac - [ -f "$scenario_path" ] && grep -q 'DUCKGRES_SCENARIO_FROZEN_S3_URI' "$scenario_path" -} - -job_name() { - local name run_hash - name="${SCENARIO_NAME:?SCENARIO_NAME is required}" - run_hash="$(printf '%s' "${DUCKGRES_SCENARIO_RUN_ID:?DUCKGRES_SCENARIO_RUN_ID is required}" | cksum | awk '{print $1}')" - printf 'duckgres-scenario-%s-%s\n' "$name" "$run_hash" | tr '_' '-' -} - -cmd_deploy() { - if [ "$USE_SHARED_DEV" = "true" ]; then - echo "Skipping isolated Duckgres deploy; using shared dev stack." - return 0 - fi - bash "$ROOT/tests/e2e-mw-dev/run.sh" deploy -} - -cmd_test() { - local ns job api_base pg flight suffix rc - : "${SCENARIO_RUNNER_IMAGE:?SCENARIO_RUNNER_IMAGE is required}" - : "${SCENARIO_FILE:?SCENARIO_FILE is required}" - : "${DUCKGRES_SCENARIO_RUN_ID:?DUCKGRES_SCENARIO_RUN_ID is required}" - : "${DUCKGRES_SCENARIO_MAX_RUNTIME:?DUCKGRES_SCENARIO_MAX_RUNTIME is required}" - : "${DUCKGRES_SCENARIO_GO_TEST_TIMEOUT:?DUCKGRES_SCENARIO_GO_TEST_TIMEOUT is required}" - - ns="$(target_namespace)" - job="$(job_name)" - api_base="$(control_plane_api_base)" - pg="$(pg_host)" - flight="$(flight_addr)" - suffix="$(sni_suffix)" - if scenario_requires_frozen_config; then - ensure_config_secret - fi - - "${KUBECTL[@]}" -n "$ns" delete job "$job" --ignore-not-found - cat </dev/null)" = "True" ]; then - echo "scenario Job complete." - return 0 - fi - if [ "$("${KUBECTL[@]}" -n "$ns" get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null)" = "True" ]; then - echo "scenario Job failed." >&2 - return 1 - fi - sleep 10 - done - echo "scenario Job did not reach a terminal state in time." >&2 - return 1 -} - -copy_artifacts() { - local ns="$1" job="$2" pod dest - pod="$("${KUBECTL[@]}" -n "$ns" get pod -l "job-name=$job" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" - [ -n "$pod" ] || return 0 - dest="$ROOT/artifacts/scenario-dev/${SCENARIO_NAME}" - mkdir -p "$dest" - "${KUBECTL[@]}" -n "$ns" cp "$pod:/artifacts/scenario-dev" "$dest" >/dev/null 2>&1 || true -} - -cmd_diagnostics() { - local ns job - ns="$(target_namespace)" - job="$(job_name)" - echo "::group::scenario job state" - "${KUBECTL[@]}" -n "$ns" get job,pod -l "job-name=$job" -o wide || true - "${KUBECTL[@]}" -n "$ns" describe job "$job" || true - "${KUBECTL[@]}" -n "$ns" describe pod -l "job-name=$job" || true - echo "::endgroup::" - if [ "$USE_SHARED_DEV" != "true" ]; then - bash "$ROOT/tests/e2e-mw-dev/run.sh" diagnostics || true - fi -} - -cmd_teardown() { - local ns job - ns="$(target_namespace)" - job="$(job_name)" - "${KUBECTL[@]}" -n "$ns" delete job "$job" --ignore-not-found --wait=false || true - if [ "$USE_SHARED_DEV" = "true" ]; then - return 0 - fi - bash "$ROOT/tests/e2e-mw-dev/run.sh" teardown -} - -case "${1:?usage: run.sh deploy|test|diagnostics|teardown}" in - deploy) cmd_deploy ;; - test) cmd_test ;; - diagnostics) cmd_diagnostics ;; - teardown) cmd_teardown ;; - *) echo "unknown: $1" >&2; exit 2 ;; -esac diff --git a/tests/scenario-dev/run_sh_test.go b/tests/scenario-dev/run_sh_test.go deleted file mode 100644 index 425dc83b..00000000 --- a/tests/scenario-dev/run_sh_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package scenariodev_test - -import ( - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -func TestDeploySharedDevSkipsIsolatedStack(t *testing.T) { - cmd := exec.Command("bash", "run.sh", "deploy") - cmd.Dir = "." - cmd.Env = append(os.Environ(), - "KUBE_CONTEXT=test-context", - "USE_SHARED_DEV=true", - ) - - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("shared deploy skip failed: %v\n%s", err, out) - } - if !strings.Contains(string(out), "Skipping isolated Duckgres deploy") { - t.Fatalf("shared deploy did not explain skip:\n%s", out) - } -} - -func TestSharedDevTestRequiresSharedConnectionConfig(t *testing.T) { - cmd := exec.Command("bash", "run.sh", "test") - cmd.Dir = "." - cmd.Env = append(os.Environ(), - "KUBE_CONTEXT=test-context", - "USE_SHARED_DEV=true", - "SCENARIO_RUNNER_IMAGE=example.invalid/duckgres:scenario", - "SCENARIO_NAME=provision-smoke", - "SCENARIO_FILE=tests/scenario/scenarios/provision_smoke.yaml", - "DUCKGRES_SCENARIO_RUN_ID=scenario-dev-test", - "DUCKGRES_SCENARIO_MAX_RUNTIME=30m", - "DUCKGRES_SCENARIO_GO_TEST_TIMEOUT=45m", - ) - - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("shared test succeeded without shared connection config:\n%s", out) - } - if !strings.Contains(string(out), "SCENARIO_SHARED_API_BASE is required") { - t.Fatalf("shared test did not report missing API base:\n%s", out) - } -} - -func TestProvisionScenarioDoesNotRequireFrozenConfigSecret(t *testing.T) { - binDir, callsPath := newFakeBin(t) - cmd := exec.Command("bash", "run.sh", "test") - cmd.Dir = "." - cmd.Env = append(os.Environ(), - "PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"), - "SCENARIO_DEV_TEST_CALLS="+callsPath, - "KUBE_CONTEXT=test-context", - "SCENARIO_NAMESPACE=duckgres-ci-pr-1231", - "SCENARIO_RUNNER_IMAGE=example.invalid/duckgres:scenario", - "SCENARIO_NAME=provision-smoke", - "SCENARIO_FILE=tests/scenario/scenarios/provision_smoke.yaml", - "DUCKGRES_SCENARIO_RUN_ID=scenario-dev-test", - "DUCKGRES_SCENARIO_MAX_RUNTIME=30m", - "DUCKGRES_SCENARIO_GO_TEST_TIMEOUT=45m", - ) - - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("provision scenario should not require frozen config secret: %v\n%s", err, out) - } - callsRaw, err := os.ReadFile(callsPath) - if err != nil { - t.Fatalf("read fake kubectl calls: %v", err) - } - calls := string(callsRaw) - if strings.Contains(calls, "get secret duckgres-scenario-config") { - t.Fatalf("provision scenario unexpectedly read frozen config secret; calls:\n%s", calls) - } - if !strings.Contains(calls, "apply -f -") { - t.Fatalf("provision scenario did not create the scenario Job; calls:\n%s", calls) - } -} - -func TestScenarioJobPinsArm64RunnerImage(t *testing.T) { - raw, err := os.ReadFile("run.sh") - if err != nil { - t.Fatalf("read run.sh: %v", err) - } - script := string(raw) - for _, required := range []string{ - "nodeSelector:", - "kubernetes.io/arch: arm64", - } { - if !strings.Contains(script, required) { - t.Fatalf("scenario job manifest missing %q", required) - } - } -} - -func TestScenarioDevScriptUsesRunSpecificJobNameAndClusterIPHostaddr(t *testing.T) { - raw, err := os.ReadFile("run.sh") - if err != nil { - t.Fatalf("read run.sh: %v", err) - } - script := string(raw) - for _, required := range []string{ - "DUCKGRES_SCENARIO_RUN_ID", - "cksum", - "jsonpath='{.spec.clusterIP}'", - "SCENARIO_ISOLATED_SNI_SUFFIX", - "SCENARIO_SHARED_SNI_SUFFIX", - } { - if !strings.Contains(script, required) { - t.Fatalf("scenario-dev script missing %q", required) - } - } -} - -func newFakeBin(t *testing.T) (string, string) { - t.Helper() - - dir := t.TempDir() - binDir := filepath.Join(dir, "bin") - if err := os.Mkdir(binDir, 0o755); err != nil { - t.Fatalf("mkdir fake bin: %v", err) - } - callsPath := filepath.Join(dir, "calls.log") - - writeFake(t, binDir, "kubectl", `#!/usr/bin/env bash -printf 'kubectl %s\n' "$*" >> "$SCENARIO_DEV_TEST_CALLS" -if [[ "$*" == *" get secret duckgres-scenario-config "* ]]; then - printf 'frozen config secret should not be required for this scenario\n' >&2 - exit 1 -fi -if [[ "$*" == *" get svc duckgres-control-plane "* ]]; then - printf '10.96.0.20' - exit 0 -fi -if [[ "$*" == *" apply -f -"* ]]; then - cat >/dev/null - exit 0 -fi -if [[ "$*" == *" get job duckgres-scenario-provision-smoke-"* ]]; then - printf 'True' - exit 0 -fi -if [[ "$*" == *" get pod -l job-name=duckgres-scenario-provision-smoke-"* ]]; then - printf 'duckgres-scenario-provision-smoke-pod' - exit 0 -fi -exit 0 -`) - writeFake(t, binDir, "date", `#!/usr/bin/env bash -printf 'date %s\n' "$*" >> "$SCENARIO_DEV_TEST_CALLS" -if [[ "$*" == "+%s" ]]; then - printf '2000000000' - exit 0 -fi -exec /bin/date "$@" -`) - writeFake(t, binDir, "sleep", `#!/usr/bin/env bash -printf 'sleep %s\n' "$*" >> "$SCENARIO_DEV_TEST_CALLS" -`) - - return binDir, callsPath -} - -func writeFake(t *testing.T, binDir, name, body string) { - t.Helper() - - path := filepath.Join(binDir, name) - if err := os.WriteFile(path, []byte(body), 0o755); err != nil { - t.Fatalf("write fake %s: %v", name, err) - } -}