test(e2e-mw-dev): port the kind k8s suite into the real-cluster harness; retire tests/k8s

.github/workflows/ci.yml

@@ -247,88 +247,3 @@ jobs:
 
       - name: Run Kubernetes-tagged control plane package tests
         run: just test-controlplane-k8s
-
-  k8s-integration-tests:
-    needs: unit-tests
-    runs-on: ubuntu-24.04-arm
-    timeout-minutes: 30
-    # id-token: write lets this job request a GitHub OIDC token, which
-    # aws-actions/configure-aws-credentials trades for STS-vended AWS
-    # credentials by assuming github-duckgres-iceberg-ci-testing-role in mw-dev.
-    # The role's trust policy is scoped to repo:PostHog/duckgres:*; its
-    # IAM policy is scoped to the iceberg test buckets only. Provisioned
-    # by PostHog/posthog-cloud-infra#8124.
-    permissions:
-      id-token: write
-      contents: read
-    env:
-      DUCKGRES_KIND_CLUSTER_NAME: duckgres
-      DUCKGRES_KIND_NODE_IMAGE: kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865
-      # Iceberg integration test (tests/k8s/iceberg_test.go) fails openly
-      # when any of these is unset — see its godoc for the rationale. The
-      # AWS_* credentials are populated by configure-aws-credentials below
-      # via OIDC; the three iceberg-specific values are bucket coordinates
-      # provisioned in mw-dev.
-      DUCKGRES_K8S_ICEBERG_TABLE_BUCKET_ARN: arn:aws:s3tables:us-east-1:${{ secrets.MW_DEV_ACCOUNT_ID }}:bucket/posthog-duckgres-iceberg-test-mw-dev
-      DUCKGRES_K8S_ICEBERG_REGION: us-east-1
-      DUCKGRES_K8S_ICEBERG_DATA_BUCKET: posthog-duckgres-iceberg-test-data-mw-dev
-
-    services:
-      postgres:
-        image: public.ecr.aws/docker/library/postgres:16-alpine
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: testdb
-        ports:
-          - 127.0.0.1:35432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Set up just
-        uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
-      - name: Set up Go
-        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
-        with:
-          go-version-file: go.mod
-      - name: Install kind
-        env:
-          KIND_VERSION: v0.24.0
-        run: |
-          curl -sSL -o /tmp/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-arm64"
-          chmod +x /tmp/kind
-          sudo mv /tmp/kind /usr/local/bin/kind
-          kind --version
-      - name: Clear Docker Hub credentials for kind pulls
-        run: |
-          docker logout registry-1.docker.io || true
-          docker logout docker.io || true
-          docker logout https://index.docker.io/v1/ || true
-      - name: Pre-pull kind node image
-        run: |
-          for attempt in 1 2 3; do
-            if docker pull "${DUCKGRES_KIND_NODE_IMAGE}"; then
-              exit 0
-            fi
-            sleep $((attempt * 5))
-          done
-          exit 1
-      - name: Configure AWS credentials via OIDC
-        # Trades the GitHub-issued OIDC token for STS credentials by
-        # assuming github-duckgres-iceberg-ci-testing-role in mw-dev. Exposes
-        # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN
-        # in the job env, which iceberg_test.go reads via os.Getenv.
-        # Pinned to the same commit cloud-infra workflows use.
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v4.0.2
-        with:
-          aws-region: us-east-1
-          role-to-assume: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/github-duckgres-iceberg-ci-testing-role
-          role-duration-seconds: 3600
-      - name: Run Kubernetes integration tests
-        run: just test-k8s-integration

.github/workflows/e2e-mw-dev.yml

@@ -51,9 +51,14 @@ on:
       - ".github/workflows/e2e-mw-dev.yml"
       - ".github/workflows/_image-build.yml"
   workflow_dispatch:
+  # Cleanup backstop: every 6h, sweep per-PR namespaces orphaned by runs that
+  # died hard (cancelled mid-flight, runner OOM) before their always() teardown
+  # could fire. Only the `e2e-cleanup` job runs on this trigger.
+  schedule:
+    - cron: "0 */6 * * *"
 
 # One in-flight run per PR; a new push cancels the old run (and its namespace
-# is GC'd by the always() teardown of the cancelled run + the janitor).
+# is GC'd by the always() teardown of the cancelled run + the e2e-cleanup sweep).
 concurrency:
   group: e2e-mw-dev-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -69,6 +74,8 @@ jobs:
   # builds that single image — not the separate worker/controlplane CD images —
   # to match what actually ships to mw-dev.
   build:
+    # PR / manual only — the scheduled trigger runs nothing but e2e-cleanup.
+    if: github.event_name != 'schedule'
     uses: ./.github/workflows/_image-build.yml
     with:
       dockerfile: Dockerfile
@@ -89,6 +96,7 @@ jobs:
 
   e2e:
     needs: [build]
+    if: github.event_name != 'schedule'
     runs-on: ubuntu-24.04-arm
     timeout-minutes: 40
     env:
@@ -157,3 +165,37 @@ jobs:
       - name: Teardown
         if: always()
         run: bash tests/e2e-mw-dev/run.sh teardown
+
+  # Backstop sweep of orphaned per-PR namespaces. Scheduled-trigger only (the
+  # PR/manual triggers run build+e2e instead). Reuses the same Tailscale + e2e
+  # IAM role as the e2e job; run.sh e2e-cleanup discovers stale namespaces by the
+  # managed-by=e2e-mw-dev label and reaps anything older than 6h. Named
+  # e2e-cleanup (not "janitor") to avoid colliding with duckgres's own janitor.
+  e2e-cleanup:
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 20
+    env:
+      KUBE_CONTEXT: posthog-mw-dev
+      CLUSTER_NAME: posthog-mw-dev
+      EKS_CLUSTER_NAME: posthog-mw-dev
+      AWS_REGION: us-east-1
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.MW_DEV_ACCOUNT_ID }}:role/github-duckgres-e2e
+          aws-region: us-east-1
+      - name: Connect to Tailscale
+        uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2
+        with:
+          oauth-client-id: ${{ vars.TS_WIF_CLIENT_ID_MW_DEV }}
+          audience: ${{ vars.TS_WIF_AUDIENCE_MW_DEV }}
+          tags: tag:github-runner
+      - name: Install kubectl
+        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4.0.1
+      - name: Update kubeconfig
+        run: aws eks update-kubeconfig --name "$CLUSTER_NAME" --region us-east-1 --alias "$KUBE_CONTEXT"
+      - name: Sweep stale per-PR namespaces
+        run: bash tests/e2e-mw-dev/run.sh e2e-cleanup

CLAUDE.md

@@ -101,24 +101,44 @@ The project uses [just](https://github.com/casey/just) as a command runner. Run
 
 ## Testing
 
+**Every feature, behavior change, bugfix, AND refactor that affects runtime or
+cluster behavior MUST ship with a solid end-to-end test case in
+`tests/e2e-mw-dev/` (`harness.sh`).** This is not just for new features — any
+change to how the system behaves at runtime (new capability, changed semantics,
+a fixed bug, a new config knob, an activation/routing/teardown tweak) extends or
+adds a harness assertion in the same PR. Refactors count too: when you move or
+rewrite a code path the harness covers, confirm the relevant assertion still
+exercises it (and update it if the path moved) — a refactor that quietly drops
+e2e coverage is a regression in the test suite even if behavior is unchanged. Unit/package tests are necessary but not sufficient: a
+change is only "done" once it is exercised against the real mw-dev cluster —
+real worker pods, real Crossplane ducklings, real cnpg/RDS metadata, real
+Lakekeeper, real S3/Iceberg/STS. "Solid" means a deterministic pass/fail
+assertion of the actual user-visible behavior (not just "it didn't error"), with
+transient/cold-pool conditions handled, on both metadata backends (cnpg + ext)
+where it touches metadata. A bugfix gets a regression assertion that would have
+caught the bug. If a change genuinely cannot be asserted in-Job (e.g. it needs
+cnpg-shards exec, or warm-pool-only state), say so explicitly in the
+harness/README with the reason — don't silently skip. The harness is the gate
+that catches what unit tests fake.
+
 Three test lanes worth knowing about, in increasing order of blast radius:
 
-- **Unit / package tests** (`go test ./...`): in-process, no external deps. Where most coverage lives.
+- **Unit / package tests** (`go test ./...`): in-process, no external deps. Where most coverage lives. Includes `tests/manifests/` (static-manifest artifact asserts for `k8s/rbac.yaml` + `k8s/networkpolicy.yaml`).
 - **`tests/integration/`** (`just test-integration`): spins up the standalone server binary against a real MinIO + Postgres metadata store via docker compose. Covers wire protocol, DuckLake on real S3-compatible storage, transpilation against a live server.
-- **`tests/k8s/`** (`just test-k8s-integration`): real kind cluster, real control-plane pod, real worker pods, real config-store Postgres. Multi-tenant activation, worker pool, Flight RPC, k8s pod lifecycle. **The iceberg test in here additionally hits real AWS S3 Tables** in the mw-dev sandbox via GitHub OIDC. See `tests/k8s/CLAUDE.md` before touching anything in there — TestMain is destructive against the current kubeconfig.
+- **`tests/e2e-mw-dev/`** (per-PR GitHub workflow `e2e-mw-dev.yml`): the full multi-tenant activation pipeline against the **real posthog-mw-dev EKS cluster** — real Cilium, real Crossplane ducklings, real cnpg-shard + external-RDS metadata, real per-org Lakekeeper, real AWS S3/Iceberg. A shell harness (`harness.sh`) runs as an in-cluster Job per PR; `run.sh` orchestrates deploy/test/teardown/e2e-cleanup. **Replaces the retired kind suite** (`tests/k8s/`) — that suite's `k8s-integration-tests` CI job and its Go tests are gone; the supporting `k8s/` scripts/manifests + Dockerfiles are kept for now. See `tests/e2e-mw-dev/README.md`.
 
 ### When code changes obligate test changes
 
-The k8s integration suite is the only place we exercise the full activation pipeline (control plane → STS broker → worker pod → DuckDB → ATTACH against real cloud storage). If your change touches any of the following, treat updating `tests/k8s/` as part of the change, not a follow-up:
+`tests/e2e-mw-dev/` is the only place we exercise the full activation pipeline (control plane → STS broker → worker pod → DuckDB → ATTACH against real cloud storage). If your change touches any of the following, treat updating the harness as part of the change, not a follow-up:
 
 - `controlplane/shared_worker_activator.go`, `controlplane/sts_broker.go`, anything in the activation payload shape (`TenantActivationPayload`, `server.DuckLakeConfig`, `server.IcebergConfig`)
 - `server/server.go::AttachDeltaCatalog`, `server.AttachIcebergCatalog`, `server.attachDuckLake*`, `server.refresh*Secret`
-- `server/iceberg/` (config, dispatcher, backend implementations) — every backend split needs a seed update in `tests/k8s/iceberg_test.go::buildIcebergConfigStoreSeed` (the iceberg_backend column default is "lakekeeper", so omitting it silently routes to the wrong path)
-- `controlplane/configstore/models.go` — new columns on `ManagedWarehouse` / `ManagedWarehouseIceberg` / sub-structs need to be set in the test seed, otherwise GORM defaults take over silently
+- `server/iceberg/` (config, dispatcher, backend implementations) — the harness provisions iceberg-enabled ducklings on both cnpg + ext backends and asserts the catalog attaches + reads/writes
+- `controlplane/configstore/models.go` — new columns flow through the provisioning API the harness calls; exercise them via a provision body field
 - `duckdbservice/activation.go`, `worker_activation.go` — worker-side activation order
 - Any code path that wires AWS credentials through to DuckDB SECRETs
 
-The contract is: if a test that already exists no longer exercises the path you changed, **update it** (don't add a new one that duplicates the setup). If your change removes a path the tests still assert against, **delete the assertion**. The DuckLake round-trip / durability / concurrent-writers tests in `tests/k8s/ducklake_test.go` and the iceberg activation test in `tests/k8s/iceberg_test.go` are the load-bearing ones for catalog wiring — keep them honest.
+The contract: if the harness no longer exercises a path you changed, **update `harness.sh`**; if your change removes a path it asserts against, **delete the assertion**. The DuckLake round-trip / durability / concurrent-writers / iceberg activation checks in `harness.sh` are the load-bearing ones for catalog wiring — keep them honest.
 
 ## Dependencies
 

tests/configstore/seed_sql_test.go

@@ -92,7 +92,7 @@ func TestKindConfigStoreSeedSQL(t *testing.T) {
 }
 
 func TestTenantIsolationConfigStoreSeedSQL(t *testing.T) {
-	seedPath := filepath.Join(findProjectRoot(t), "tests", "k8s", "testdata", "tenant-isolation.seed.sql")
+	seedPath := filepath.Join(findProjectRoot(t), "tests", "configstore", "testdata", "tenant-isolation.seed.sql")
 	seedSQL, err := os.ReadFile(seedPath)
 	if err != nil {
 		t.Fatalf("read tenant isolation seed: %v", err)

tests/e2e-mw-dev/README.md

@@ -22,6 +22,69 @@ layers where this quarter's production bugs lived.
 5. **Teardown** always: deprovision the ci-pr ducklings (clean shared-infra
    footprint) then delete the namespace.
 
+A scheduled (`cron`) **e2e-cleanup** job (`run.sh e2e-cleanup`) runs every 6h and
+reaps any `duckgres-ci-pr-*` namespace older than 6h — a backstop for runs that
+died hard before their `always()` teardown could fire. (Named e2e-cleanup, not
+"janitor", to avoid colliding with duckgres's own control-plane janitor.)
+
+## What the harness asserts (`harness.sh`)
+
+This suite is the **successor to the retired kind suite** (`tests/k8s/`): every
+behavior that suite asserted against a fake kind cluster is re-asserted here
+against real mw-dev, on both the cnpg and external-RDS metadata backends. The
+in-cluster Job runs as the `duckgres` SA and uses `kubectl` (in-cluster config
+from its mounted SA token) for the pod-level checks the Go suite made via
+client-go:
+
+- **wire/query** — `SELECT 1` round-trips; 5 concurrent connections stay
+  distinct (ported from `TestK8sMultipleConcurrentConnections`).
+- **warm-pool backpressure** — a cold-pool burst of sessions outruns the worker
+  pool (`shared_warm_target=0`); the CP must answer the surplus with the
+  graceful client-visible `no warm Duckgres worker … retry in about 45 seconds`
+  hint (not a hang/500/drop), and the pool must then drain so a retrying
+  connection succeeds. The harness asserts the hint **and** handles it (the
+  concurrency tests retry through it).
+- **activation** — DuckLake **and** Iceberg catalogs attach and read/write.
+- **extension forks** — the bundled `ducklake`/`httpfs` extensions are the
+  PostHog forks, not upstream (ported from the `*IsBundledFork` tests).
+- **worker pods** — labels (`app`, `duckgres/control-plane`,
+  `duckgres/worker-id`), securityContext (`runAsNonRoot`, uid 1000, no
+  priv-esc), Downward-API `POD_NAME`/`NODE_NAME` env, and **no** ambient
+  SA-token mount.
+- **resilience** — worker-pod kill → crash recovery; DuckLake durability across
+  a worker restart; concurrent writers (fork conflict-retry, the test that was
+  flaking on main).
+- **isolation** — two tenants (cnpg vs ext) see distinct catalogs; a
+  cross-tenant read is denied.
+- **lifecycle** — deprovision → `warehouse=deleted` → the Crossplane Duckling
+  CR **fully** deletes (`kubectl wait --for=delete`, asserting the finalizer
+  cascade that drops the cnpg role+db completed). Same-id **re-provision** is
+  *not* done in-Job: a clean slate needs DROPping a possibly-stranded cnpg role,
+  which only `run.sh` (on the runner, with cnpg-shards exec) can do — so the
+  stranded-cnpg-role regression (#649/#650/#11518/#11522) is covered **across
+  runs** (`run.sh deploy` drops the role for a clean slate; `run.sh teardown`
+  waits the CR `--for=delete`), not within one Job.
+
+**Static-manifest asserts** (`k8s/rbac.yaml`, `k8s/networkpolicy.yaml`) that the
+kind suite carried as unit tests now live in `tests/manifests/` and run in the
+normal `go test ./...` lane.
+
+### Deliberately not covered here
+
+- **Shared-warm-worker activation** and the **version-mismatch idle-worker
+  reaper** — the per-PR CP runs `DUCKGRES_K8S_SHARED_WARM_TARGET=0`, so there
+  are no idle warm workers to assert on. These stay covered by `controlplane/`
+  unit tests; running them end-to-end would need a warm target >0 in the per-PR
+  CP.
+- **Physical object-store-prefix isolation** — the Go suite listed the MinIO
+  prefix to prove writes land only in a tenant's own path. Against real mw-dev
+  S3 the Job holds no list creds, so isolation is asserted **logically** (the
+  cross-tenant read is denied) rather than by enumerating S3 objects.
+- **Cilium egress allow/deny probing** — asserting a worker reaches the cnpg
+  pooler + `lakekeeper:8181` but not a denied destination needs a stable
+  exec-into-worker probe; deferred (high flake risk). The policies themselves
+  are asserted statically in `tests/manifests/`.
+
 ## Isolation model
 
 Dedicated CP + throwaway config-store **per PR**, provisioning **real**
@@ -105,22 +168,23 @@ id committed).
   failed auths (~15 min). The harness uses the provision-time password and
   settles one config-poll interval before connecting — keep it that way; a
   reset-password + tight retry loop will trip the ban.
-- **Teardown is async-incomplete.** `run.sh teardown` deprovisions and waits on
-  warehouse `state=deleted` (= Duckling CR deleted), but the downstream
-  Crossplane DROP of the cnpg role+db lags behind that — observed a stranded
-  `lakekeeper_ci_pr_<N>_cnpg` role after teardown returned. Either wait on the
-  cnpg role/db actually being gone, or rely on the janitor below. (The
-  composition `managementPolicies: ["*"]` from charts#11522 does drop them;
-  it's just not synchronous with the CR delete.)
+- **Teardown / recreate are now CR-synchronous.** `run.sh teardown` and the
+  in-harness same-org recreate both `kubectl wait --for=delete` on the Duckling
+  CR, whose finalizers run the Crossplane DROP of the cnpg role+db, before
+  returning / re-provisioning. `drop_cnpg_role` is still called at deploy + at
+  teardown as a belt-and-suspenders idempotent backstop. (Composition
+  `managementPolicies: ["*"]` from charts#11522 does the drop; the `--for=delete`
+  wait is what makes it synchronous from our side.)
 - **Shared-infra contention.** Concurrent PRs provision real ducklings against
   the same cnpg-shards / RDS / lakekeeper-operator. Org-ID prefix keeps them
   distinct; watch quay.io / cnpg pooler / RDS limits under parallelism.
-- **Deep coverage stubbed.** Durability (worker-pod kill), concurrency, and
-  network-policy assertions are `SKIP (TODO)` — activation (#6) is cleared, so
-  these are the next layer to flesh out.
-- **Janitor.** Periodic sweep of stale `duckgres-ci-pr-*` namespaces + their
-  ci-pr-labelled cross-ns bindings + orphaned ducklings/cnpg roles, to back up
-  the always() teardown for runs that die hard. Not yet added.
+- **e2e-cleanup** is wired: the `e2e-mw-dev.yml` `schedule` trigger runs
+  `run.sh e2e-cleanup` every 6h, reaping `duckgres-ci-pr-*` namespaces older than
+  6h (`E2E_CLEANUP_MAX_AGE_HOURS`) along with their ducklings, cnpg role+db, Pod
+  Identity association, and ci-pr-labelled cross-ns bindings.
+- **Remaining deferrals** are listed under "Deliberately not covered here"
+  above (warm-pool activation + version-reaper, physical S3-prefix isolation,
+  Cilium egress allow/deny probing).
 
 ## Local dry-run