From 29740b4fcfaf2075fe1a458e5f4ac2a472ab3972 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Thu, 9 Jul 2026 13:06:39 +0200 Subject: [PATCH] Reshard form: discover all destination shards and known external stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The form's shard dropdown was derived from duckling statuses — the shards tenants already OCCUPY — so a freshly created empty shard (mw-dev shard-002) never appeared, exactly the shard a rebalancing reshard wants to target. New GET /reshards/targets: cnpg shards discovered from the CNPG instance pods (label cnpg.io/cluster, cnpg-shards namespace) via the same cluster-topology read the Nodes view uses — includes EMPTY shards; an RBAC Forbidden (the e2e CP) degrades to the occupied-shard fallback with cluster_discovery=false so the form can say why a new shard needs manual entry. Plus known external stores: distinct external metadata stores of live warehouses (endpoint + SM secret NAME + user/db — never a password), offered as prefill for the cnpg->ext escape hatch. UI: shard dropdown fed by the endpoint (unioned with occupied shards as fallback), a degrade notice, and a known-external-stores prefill select; the password stays manual always. --- controlplane/admin/reshard.go | 94 +++++++++++++++++-- controlplane/admin/reshard_test.go | 87 ++++++++++++++++- controlplane/admin/ui/src/hooks/useApi.ts | 20 ++++ controlplane/admin/ui/src/lib/api.ts | 4 + .../admin/ui/src/pages/ReshardForm.tsx | 51 +++++++++- controlplane/admin/ui/src/types/api.ts | 18 ++++ controlplane/configstore/reshard.go | 28 ++++++ controlplane/multitenant.go | 2 +- docs/design/resharding.md | 2 +- tests/configstore/reshard_postgres_test.go | 38 ++++++++ tests/e2e-mw-dev/harness.sh | 19 +++- 11 files changed, 348 insertions(+), 15 deletions(-) diff --git a/controlplane/admin/reshard.go b/controlplane/admin/reshard.go index 857dbc9b..91cd3828 100644 --- a/controlplane/admin/reshard.go +++ b/controlplane/admin/reshard.go @@ -7,11 +7,15 @@ import ( "errors" "net/http" "regexp" + "sort" "strconv" "strings" "github.com/gin-gonic/gin" "gorm.io/gorm" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "github.com/posthog/duckgres/controlplane/configstore" ) @@ -37,6 +41,7 @@ type ReshardStore interface { FinishPendingReshardOperation(id int64, state configstore.ReshardState, errMsg string) (bool, error) AppendReshardLog(opID int64, level, message string) error GetManagedWarehouse(orgID string) (*configstore.ManagedWarehouse, error) + ListExternalMetadataStores() ([]configstore.ExternalMetadataStoreInfo, error) } // ReshardPasswordStash hands the runner the ephemeral external password for @@ -73,20 +78,97 @@ type startReshardRequest struct { // RegisterReshardAPI wires the reshard endpoints. lister may be nil (duckling // client unavailable) — starting a reshard then 503s; reads still work. -// stash may be nil (no local runner) — external targets then 503. -func RegisterReshardAPI(r *gin.RouterGroup, store ReshardStore, lister DucklingMetadataLister, stash ReshardPasswordStash) { - h := &reshardHandler{store: store, lister: lister, stash: stash} +// stash may be nil (no local runner) — external targets then 503. cluster may +// be nil (non-k8s) — shard discovery then falls back to the shards tenants +// already occupy. +func RegisterReshardAPI(r *gin.RouterGroup, store ReshardStore, lister DucklingMetadataLister, stash ReshardPasswordStash, cluster kubernetes.Interface) { + h := &reshardHandler{store: store, lister: lister, stash: stash, cluster: cluster} r.POST("/orgs/:id/reshard", h.start) r.GET("/orgs/:id/reshards", h.listForOrg) r.GET("/reshards/:opid", h.get) r.GET("/reshards/:opid/log", h.log) + r.GET("/reshards/targets", h.targets) r.POST("/reshards/:opid/cancel", h.cancel) } type reshardHandler struct { - store ReshardStore - lister DucklingMetadataLister - stash ReshardPasswordStash + store ReshardStore + lister DucklingMetadataLister + stash ReshardPasswordStash + cluster kubernetes.Interface +} + +// cnpgShardsNamespace is where the shared CNPG metadata shards run; instance +// pods carry the operator's cnpg.io/cluster= label. +const cnpgShardsNamespace = "cnpg-shards" + +// targets returns everything the reshard form can offer as a destination: +// every cnpg shard (including EMPTY ones no tenant occupies yet — discovered +// from the CNPG instance pods via the cluster-topology read the Nodes view +// already uses; an RBAC Forbidden degrades to the shards tenants occupy, read +// from the duckling statuses) and every external metadata store a live +// warehouse references (endpoint + SM secret NAME only — never a password). +func (h *reshardHandler) targets(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), ducklingMetadataTimeout) + defer cancel() + + shardSet := map[string]struct{}{} + clusterAvailable := false + if h.cluster != nil { + pods, err := h.cluster.CoreV1().Pods(cnpgShardsNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "cnpg.io/cluster", + }) + switch { + case err == nil: + clusterAvailable = true + for i := range pods.Items { + if shard := pods.Items[i].Labels["cnpg.io/cluster"]; shard != "" { + shardSet[shard] = struct{}{} + } + } + case apierrors.IsForbidden(err): + // Same degrade contract as the cluster topology endpoints: the + // e2e CP has no cluster-scoped RBAC; fall back to occupied shards. + default: + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + } + if h.lister != nil { + stores, err := h.lister.CRMetadataStores(ctx) + if err == nil { + for _, ms := range stores { + if ms.Kind == configstore.MetadataStoreKindCnpgShard { + if shard := cnpgShardFromEndpoint(ms.Endpoint); shard != "" { + shardSet[shard] = struct{}{} + } + } + } + } + } + shards := make([]string, 0, len(shardSet)) + for s := range shardSet { + shards = append(shards, s) + } + sort.Strings(shards) + + external, err := h.store.ListExternalMetadataStores() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if external == nil { + external = []configstore.ExternalMetadataStoreInfo{} + } + + c.JSON(http.StatusOK, gin.H{ + "shards": shards, + // cluster_discovery=false means the shard list only contains shards + // tenants already occupy (RBAC degrade / non-k8s) — a brand-new empty + // shard would be missing and needs manual entry. + "cluster_discovery": clusterAvailable, + "external_stores": external, + }) } func (h *reshardHandler) start(c *gin.Context) { diff --git a/controlplane/admin/reshard_test.go b/controlplane/admin/reshard_test.go index 4fd25d27..1992c26d 100644 --- a/controlplane/admin/reshard_test.go +++ b/controlplane/admin/reshard_test.go @@ -13,6 +13,14 @@ import ( "github.com/gin-gonic/gin" "gorm.io/gorm" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "github.com/posthog/duckgres/controlplane/configstore" ) @@ -113,6 +121,12 @@ func (f *fakeReshardStore) GetManagedWarehouse(orgID string) (*configstore.Manag return &cp, nil } +func (f *fakeReshardStore) ListExternalMetadataStores() ([]configstore.ExternalMetadataStoreInfo, error) { + return []configstore.ExternalMetadataStoreInfo{ + {Endpoint: "known.rds.example.com", PasswordAWSSecret: "known-secret", User: "postgres", Database: "postgres"}, + }, nil +} + func (f *fakeReshardStore) StashExternalPassword(opID int64, password string) { if f.stashed == nil { f.stashed = map[int64]string{} @@ -129,12 +143,16 @@ func (f fakeShardLister) CRMetadataStores(context.Context) (map[string]DucklingM } func reshardRouter(store *fakeReshardStore) *gin.Engine { + return reshardRouterWithCluster(store, nil) +} + +func reshardRouterWithCluster(store *fakeReshardStore, cluster kubernetes.Interface) *gin.Engine { gin.SetMode(gin.TestMode) r := gin.New() lister := fakeShardLister{stores: map[string]DucklingMetadataStore{ "acme": {Kind: "cnpg-shard", Endpoint: "shard-001-pooler.cnpg-shards.svc.cluster.local"}, }} - RegisterReshardAPI(r.Group("/api/v1"), store, lister, store) + RegisterReshardAPI(r.Group("/api/v1"), store, lister, store, cluster) return r } @@ -303,3 +321,70 @@ func TestReshardGetListLogCancel(t *testing.T) { func itoa(n int64) string { return strconv.FormatInt(n, 10) } + +func cnpgPod(name, shard string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: cnpgShardsNamespace, + Labels: map[string]string{"cnpg.io/cluster": shard}, + }} +} + +type reshardTargets struct { + Shards []string `json:"shards"` + ClusterDiscovery bool `json:"cluster_discovery"` + ExternalStores []configstore.ExternalMetadataStoreInfo `json:"external_stores"` +} + +// TestReshardTargetsClusterDiscovery pins the load-bearing property of the +// endpoint: an EMPTY shard (no tenant on it yet) shows up, discovered from the +// CNPG instance pods, unioned with the occupied shards from duckling statuses. +func TestReshardTargetsClusterDiscovery(t *testing.T) { + cs := k8sfake.NewSimpleClientset( + cnpgPod("shard-001-1", "shard-001"), + cnpgPod("shard-002-1", "shard-002"), // empty shard: no duckling on it + ) + w := doJSON(reshardRouterWithCluster(newFakeReshardStore(), cs), http.MethodGet, "/api/v1/reshards/targets", "") + if w.Code != http.StatusOK { + t.Fatalf("status = %d body %s", w.Code, w.Body.String()) + } + var resp reshardTargets + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !resp.ClusterDiscovery { + t.Fatal("cluster_discovery = false, want true") + } + if strings.Join(resp.Shards, ",") != "shard-001,shard-002" { + t.Fatalf("shards = %v, want [shard-001 shard-002]", resp.Shards) + } + if len(resp.ExternalStores) != 1 || resp.ExternalStores[0].Endpoint != "known.rds.example.com" { + t.Fatalf("external stores = %+v", resp.ExternalStores) + } +} + +// TestReshardTargetsDegrades pins the fallbacks: an RBAC Forbidden (the e2e +// CP) degrades to the occupied shards with cluster_discovery=false, and a nil +// cluster client behaves the same. +func TestReshardTargetsDegrades(t *testing.T) { + cs := k8sfake.NewSimpleClientset() + cs.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "", nil) + }) + for _, cluster := range []kubernetes.Interface{cs, nil} { + w := doJSON(reshardRouterWithCluster(newFakeReshardStore(), cluster), http.MethodGet, "/api/v1/reshards/targets", "") + if w.Code != http.StatusOK { + t.Fatalf("status = %d body %s", w.Code, w.Body.String()) + } + var resp reshardTargets + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if resp.ClusterDiscovery { + t.Fatal("cluster_discovery = true, want false on degrade") + } + if strings.Join(resp.Shards, ",") != "shard-001" { + t.Fatalf("shards = %v, want the occupied [shard-001] fallback", resp.Shards) + } + } +} diff --git a/controlplane/admin/ui/src/hooks/useApi.ts b/controlplane/admin/ui/src/hooks/useApi.ts index 1036b73b..4de9f20b 100644 --- a/controlplane/admin/ui/src/hooks/useApi.ts +++ b/controlplane/admin/ui/src/hooks/useApi.ts @@ -38,6 +38,7 @@ import type { QueryDetail, ReshardLogEntry, ReshardOperation, + ReshardTargetsResponse, RunningQuery, SessionStatus, StartReshardBody, @@ -527,6 +528,25 @@ export function useReshardLog(opId: number | null, opState: string | undefined) return entries; } +const EMPTY_RESHARD_TARGETS: ReshardTargetsResponse = { + shards: [], + cluster_discovery: false, + external_stores: [], +}; + +// Destination discovery for the reshard form. Pre-rollout backends without +// the endpoint degrade to empty (the form still allows manual entry). +export function useReshardTargets() { + return useQuery({ + queryKey: ["reshardTargets"], + queryFn: () => + tolerateStatus(EMPTY_RESHARD_TARGETS, 403, 404, 503)( + api.getReshardTargets(), + ).catch(() => EMPTY_RESHARD_TARGETS), + refetchInterval: POLL.slow, + }); +} + export function useCancelReshard() { const qc = useQueryClient(); return useMutation({ diff --git a/controlplane/admin/ui/src/lib/api.ts b/controlplane/admin/ui/src/lib/api.ts index 49afa464..a35ca19d 100644 --- a/controlplane/admin/ui/src/lib/api.ts +++ b/controlplane/admin/ui/src/lib/api.ts @@ -32,6 +32,7 @@ import type { QueryResult, ReshardLogEntry, ReshardOperation, + ReshardTargetsResponse, RunningQuery, SessionStatus, StartReshardBody, @@ -237,4 +238,7 @@ export const api = { ), cancelReshard: (opId: number) => post<{ cancel_requested?: boolean; state?: string }>(`/reshards/${opId}/cancel`, {}), + // Destination discovery for the reshard form: all cnpg shards (incl. empty + // ones) + known external stores. + getReshardTargets: () => get("/reshards/targets"), }; diff --git a/controlplane/admin/ui/src/pages/ReshardForm.tsx b/controlplane/admin/ui/src/pages/ReshardForm.tsx index f55285f6..a8031163 100644 --- a/controlplane/admin/ui/src/pages/ReshardForm.tsx +++ b/controlplane/admin/ui/src/pages/ReshardForm.tsx @@ -24,7 +24,7 @@ import { SelectValue, } from "@/components/ui/select"; import { ducklingEntryFor } from "@/lib/format"; -import { useDucklingsMetadata, useStartReshard, useWarehouse } from "@/hooks/useApi"; +import { useDucklingsMetadata, useReshardTargets, useStartReshard, useWarehouse } from "@/hooks/useApi"; import type { StartReshardBody } from "@/types/api"; // Reshard start form: pick a target metadata store for the org, confirm, run. @@ -55,15 +55,20 @@ export function ReshardForm() { const sourceKind = entry?.kind ?? warehouse.data?.metadata_store?.kind ?? "cnpg-shard"; const currentShard = entry?.cnpg_shard ?? ""; - // Known shards: the distinct set the fleet is already using. Free-text - // covers a brand-new shard no tenant landed on yet. + // Destination discovery: every cnpg shard in the cluster (including EMPTY + // ones no tenant occupies yet) + known external stores, from + // /reshards/targets. Unioned with the shards tenants occupy (duckling + // metadata) as a belt-and-suspenders fallback; free-text still covers a + // shard the server can't see (cluster_discovery=false on an RBAC degrade). + const targets = useReshardTargets(); const knownShards = useMemo(() => { - const s = new Set(); + const s = new Set(targets.data?.shards ?? []); for (const e of Object.values(metadata.data?.entries ?? {})) { if (e.kind === "cnpg-shard" && e.cnpg_shard) s.add(e.cnpg_shard); } return [...s].sort(); - }, [metadata.data]); + }, [targets.data, metadata.data]); + const knownExternal = targets.data?.external_stores ?? []; const effectiveShard = shard === "__custom__" ? shardCustom.trim() : shard; const targetLabel = @@ -180,6 +185,12 @@ export function ReshardForm() { + {targets.data && !targets.data.cluster_discovery && ( +

+ Shard list limited to shards tenants already occupy (no cluster read access) — + use "other…" for a new empty shard. +

+ )} {shard === "__custom__" && (
@@ -194,6 +205,36 @@ export function ReshardForm() {
) : (
+ {knownExternal.length > 0 && ( +
+ + +

+ Prefills endpoint/secret/user/database from an external store another + warehouse already uses — the password below is still required. +

+
+ )}
diff --git a/controlplane/admin/ui/src/types/api.ts b/controlplane/admin/ui/src/types/api.ts index 095943a4..37f23e60 100644 --- a/controlplane/admin/ui/src/types/api.ts +++ b/controlplane/admin/ui/src/types/api.ts @@ -528,3 +528,21 @@ export interface StartReshardBody { // duckling to converge before rolling back. cutover_timeout_seconds?: number; } + +// GET /api/v1/reshards/targets — everything the reshard form can offer as a +// destination. cluster_discovery=false means the shard list only holds shards +// tenants already occupy (RBAC degrade) — a brand-new empty shard needs +// manual entry then. +export interface ReshardTargetsResponse { + shards: string[]; + cluster_discovery: boolean; + external_stores: ExternalMetadataStoreInfo[]; +} + +// One known external metadata store (SM secret NAME only — never a password). +export interface ExternalMetadataStoreInfo { + endpoint: string; + password_aws_secret: string; + user: string; + database: string; +} diff --git a/controlplane/configstore/reshard.go b/controlplane/configstore/reshard.go index 2bc0ba9b..148ef3a1 100644 --- a/controlplane/configstore/reshard.go +++ b/controlplane/configstore/reshard.go @@ -393,3 +393,31 @@ func (cs *ConfigStore) OrgConnectionDrainState(orgID string) (OrgConnectionDrain } return status, nil } + +// ExternalMetadataStoreInfo is one distinct external Postgres metadata store +// currently referenced by a live managed warehouse — offered by the reshard +// form as a known cnpg→external target (endpoint + SM secret NAME only; the +// password is never stored anywhere in duckgres). +type ExternalMetadataStoreInfo struct { + Endpoint string `json:"endpoint"` + PasswordAWSSecret string `json:"password_aws_secret"` + User string `json:"user"` + Database string `json:"database"` +} + +// ListExternalMetadataStores returns the distinct external metadata stores of +// non-deleted warehouses, ordered by endpoint. +func (cs *ConfigStore) ListExternalMetadataStores() ([]ExternalMetadataStoreInfo, error) { + var stores []ExternalMetadataStoreInfo + err := cs.db.Model(&ManagedWarehouse{}). + Select("DISTINCT metadata_store_endpoint AS endpoint, metadata_store_password_aws_secret AS password_aws_secret, metadata_store_username AS user, metadata_store_database_name AS database"). + Where("metadata_store_kind = ? AND metadata_store_endpoint <> '' AND state NOT IN ?", + MetadataStoreKindExternal, + []ManagedWarehouseProvisioningState{ManagedWarehouseStateDeleting, ManagedWarehouseStateDeleted}). + Order("endpoint"). + Scan(&stores).Error + if err != nil { + return nil, fmt.Errorf("list external metadata stores: %w", err) + } + return stores, nil +} diff --git a/controlplane/multitenant.go b/controlplane/multitenant.go index d9e9fc2f..f2242ff4 100644 --- a/controlplane/multitenant.go +++ b/controlplane/multitenant.go @@ -577,7 +577,7 @@ func SetupMultiTenant( if reshardRunner != nil { reshardStash = reshardRunner } - admin.RegisterReshardAPI(api, store, ducklingMetadata, reshardStash) + admin.RegisterReshardAPI(api, store, ducklingMetadata, reshardStash, clusterClient) // Break-glass internal-secret login (the SPA owns "/" and app routes). admin.RegisterLogin(engine, adminTokens) diff --git a/docs/design/resharding.md b/docs/design/resharding.md index 75875647..f633a6fb 100644 --- a/docs/design/resharding.md +++ b/docs/design/resharding.md @@ -21,7 +21,7 @@ those stay valid because the bucket is untouched. | Runner (claims + executes ops) | `controlplane/provisioner/reshard_runner.go` | | Catalog copier (schema + rows + verify) | `controlplane/provisioner/catalog_copy.go` | | Duckling CR patches (flip, compaction pause) | `controlplane/provisioner/k8s_client.go` | -| Admin REST API | `controlplane/admin/reshard.go` | +| Admin REST API (start/read/cancel + `GET /reshards/targets` destination discovery: all cnpg shards incl. empty ones via the cluster-topology pod read, RBAC-degrading to occupied shards; known external stores from live warehouse rows) | `controlplane/admin/reshard.go` | | Console UI (form + operation page with live log) | `admin/ui/src/pages/ReshardForm.tsx`, `ReshardOperation.tsx` | | Connection gates | `controlplane/control.go` (57P03), `flight_ingress.go`, grant-path check in `configstore/org_connections.go` | diff --git a/tests/configstore/reshard_postgres_test.go b/tests/configstore/reshard_postgres_test.go index 0790a763..f85cfecc 100644 --- a/tests/configstore/reshard_postgres_test.go +++ b/tests/configstore/reshard_postgres_test.go @@ -325,3 +325,41 @@ func TestListWorkerRecordsForOrgPostgres(t *testing.T) { t.Fatalf("got %d records, want 2 (hot + hot_idle; terminal states excluded)", len(records)) } } + +// TestListExternalMetadataStoresPostgres pins the reshard form's external +// target discovery: distinct external stores of live warehouses only. +func TestListExternalMetadataStoresPostgres(t *testing.T) { + store := newIsolatedConfigStore(t) + + seed := func(org, endpoint string, state cpconfigstore.ManagedWarehouseProvisioningState) { + t.Helper() + if err := store.DB().Create(&cpconfigstore.Org{Name: org, DatabaseName: org}).Error; err != nil { + t.Fatalf("seed org %s: %v", org, err) + } + wh := &cpconfigstore.ManagedWarehouse{OrgID: org, DucklingName: org, State: state} + wh.MetadataStore.Kind = cpconfigstore.MetadataStoreKindExternal + wh.MetadataStore.Endpoint = endpoint + wh.MetadataStore.PasswordAWSSecret = "secret-" + org + wh.MetadataStore.Username = "postgres" + wh.MetadataStore.DatabaseName = "postgres" + if err := store.DB().Create(wh).Error; err != nil { + t.Fatalf("seed warehouse %s: %v", org, err) + } + } + seed("ext-a", "a.rds.example.com", cpconfigstore.ManagedWarehouseStateReady) + seed("ext-b", "b.rds.example.com", cpconfigstore.ManagedWarehouseStateReady) + seed("ext-gone", "gone.rds.example.com", cpconfigstore.ManagedWarehouseStateDeleted) + // A cnpg warehouse must not appear. + seedReadyWarehouse(t, store, "cnpg-org") + + stores, err := store.ListExternalMetadataStores() + if err != nil { + t.Fatalf("list: %v", err) + } + if len(stores) != 2 || stores[0].Endpoint != "a.rds.example.com" || stores[1].Endpoint != "b.rds.example.com" { + t.Fatalf("stores = %+v, want a + b only (deleted + cnpg excluded)", stores) + } + if stores[0].PasswordAWSSecret != "secret-ext-a" || stores[0].User != "postgres" || stores[0].Database != "postgres" { + t.Fatalf("store[0] = %+v", stores[0]) + } +} diff --git a/tests/e2e-mw-dev/harness.sh b/tests/e2e-mw-dev/harness.sh index a1788fc0..ed122cf2 100755 --- a/tests/e2e-mw-dev/harness.sh +++ b/tests/e2e-mw-dev/harness.sh @@ -2255,6 +2255,22 @@ reshard_dump_log() { # opid | jq -r '.entries[] | "\(.level): \(.message)"' | tail -30 || true } +reshard_targets() { # destination discovery; ext org's RDS must appear too + log "reshard targets: destination discovery" + out="$(curl -fsS -H "$H" "$API/api/v1/reshards/targets")" \ + || fail "reshard targets: request failed" + # The e2e CP has no cluster-scoped RBAC (Forbidden degrade), so the shard + # list is the occupied fallback — the cnpg org's shard must be in it. On a + # real deployment cluster_discovery=true additionally surfaces EMPTY shards. + echo "$out" | jq -e '.shards | index("shard-001") != null' >/dev/null \ + || fail "reshard targets: shard-001 missing: $out" + echo "$out" | jq -e '.external_stores | length >= 1 and (.[0].endpoint | length > 0) and (.[0].password_aws_secret | length > 0)' >/dev/null \ + || fail "reshard targets: external store from the ext org missing: $out" + echo "$out" | jq -e '.external_stores | all(has("password") | not)' >/dev/null \ + || fail "reshard targets: response carries a password field: $out" + log "reshard targets OK ($(echo "$out" | jq -c '{shards, cluster_discovery}'))" +} + reshard_validation() { # cnpg-org currently on shard-001 log "reshard validation: same-shard + bad targets are 400" for body in \ @@ -2842,6 +2858,7 @@ main() { # ---- reshard operations (validation, cancel, rollback on res2; then the # ext→cnpg POSITIVE path on the ext org — must run AFTER every assert # that needs the ext org on the RDS, notably tenant_isolation) ---- + reshard_targets reshard_validation "$CNPG" reshard_cancel_during_drain "$RES2" "$res2_pw" reshard_bogus_shard_rollback "$RES2" "$res2_pw" @@ -2856,7 +2873,7 @@ main() { # mid-run image bump); it stays covered by the controlplane/ unit tests. log "SKIP version-reaper (needs an in-run image bump; see README)" - log "PASS: admin-no-query-token + models-explorer-api(redaction) + admin-console-api(me/live/metrics/auth-gate) + admin-rbac-viewer(403 mutate/audit) + admin-impersonation(round-trip+audit) + wire + malformed-startup-resilience + jsonb-concat + cold-burst-absorption + pipeline-error-recovery + cancel-reuse + activation(DuckLake) + ducklake-explain + ext-forks + worker-pod + concurrency + durability + crash-recovery + busy-only-do-not-disrupt + graceful-drain + one-session-per-worker + parallel-cold-burst-ramp + worker-sizing(cnpg DuckLake) + org-default-profile(ext) + persistent-user-secrets(cnpg+ext, cross-user isolation) + user-kill-switch(cnpg) + user-disable-block(cnpg+ext) + connection-duration-logged + compute-usage-pull-api(cnpg, compute+storage) + duckling-shard-backfill(cnpg) + isolation + reshard(validation + cancel-during-drain + bogus-shard-rollback + ext-to-cnpg positive path) + lifecycle-teardown(+org-delete/name-release), on cnpg & ext (4 parallel lanes)" + log "PASS: admin-no-query-token + models-explorer-api(redaction) + admin-console-api(me/live/metrics/auth-gate) + admin-rbac-viewer(403 mutate/audit) + admin-impersonation(round-trip+audit) + wire + malformed-startup-resilience + jsonb-concat + cold-burst-absorption + pipeline-error-recovery + cancel-reuse + activation(DuckLake) + ducklake-explain + ext-forks + worker-pod + concurrency + durability + crash-recovery + busy-only-do-not-disrupt + graceful-drain + one-session-per-worker + parallel-cold-burst-ramp + worker-sizing(cnpg DuckLake) + org-default-profile(ext) + persistent-user-secrets(cnpg+ext, cross-user isolation) + user-kill-switch(cnpg) + user-disable-block(cnpg+ext) + connection-duration-logged + compute-usage-pull-api(cnpg, compute+storage) + duckling-shard-backfill(cnpg) + isolation + reshard(targets-discovery + validation + cancel-during-drain + bogus-shard-rollback + ext-to-cnpg positive path) + lifecycle-teardown(+org-delete/name-release), on cnpg & ext (4 parallel lanes)" } main "$@"