fix(webapp,run-ops-database): keep run-ops batch items co-resident with their batch (#4178)

## Summary

Three fixes to the run-ops database split (the Cloud-only mode where
run-lifecycle rows live on a dedicated Postgres). All are inert in the
default single-database deployment.

The main fix: on the batch trigger paths, a parentless batch's item runs
chose their physical store from a fresh per-org mint-flag read at
processing time, so flipping an org's flag mid-batch could land an item
in a different store than its batch, breaking the `TaskRun.batchId`
foreign key (or silently orphaning the item). The other two harden the
split's safety nets: the schema-parity test now actually compares
columns, and the read fan-out gate now signals when it has been silently
disabled.

## Batch item residency

`RunEngineBatchTriggerService` (api.v2) and the BatchQueue item callback
(api.v3) now anchor each item's id mint on the batch's own friendlyId,
mirroring the already-safe `BatchTriggerV3Service`. Residency is a pure
id-shape check, so an item can no longer diverge from its batch across a
mid-batch flag flip. The pre-failed-run fallback is anchored the same
way (it also sets `batchId`), and the shared mint branch is consolidated
into one helper so every mint path stays in lockstep. No new database
queries; single-database mode is unchanged (a cuid-shaped batch
friendlyId yields a cuid item).

## Schema parity test

The parity test previously read only the dedicated schema and matched
model headers with regexes, so it never compared columns and could not
catch a run-subgraph column that diverged between the two physical
schemas. It now parses both schemas and asserts bidirectional
scalar-column parity (type, nullability, array-ness, default) across the
run-subgraph models, and fails on any field line it can't parse. Scoped
to the run-subgraph models so unrelated control-plane edits don't break
it.

## Read fan-out signal

The split read fan-out gate is decided by the object identity of the NEW
vs control-plane clients. It now warns when both run-ops URLs are set
but the NEW client isn't a distinct instance (fan-out silently off), and
a new test exercises the real topology-into-gate wiring so a future
refactor that aliases the clients can't disable fan-out unnoticed.

## Verification

New unit and glue tests cover all three changes; the DB-backed
residency, store-routing, and topology suites pass against real
Postgres; `typecheck` is clean for both packages.

Author: d-cs

.server-changes/fix-batch-item-mint-anchor.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Anchor batch item run-ops residency on the batch's own friendlyId (not a fresh per-org flag read) in the run-engine batch trigger service and the BatchQueue item callback, on both the success path and the pre-failed-run failure path, so a mid-batch mint-flag flip can no longer mint an item (or a pre-failed item) into a different physical store than its BatchTaskRun row.

apps/webapp/app/db.server.ts

@@ -290,6 +290,7 @@ export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({
   controlPlaneReplica: $replica,
   hasNewUrl: !!env.RUN_OPS_DATABASE_URL,
   hasLegacyUrl: !!env.RUN_OPS_LEGACY_DATABASE_URL,
+  logger,
 });
 
 // Boot-time interlock: if the flag is on but the distinct-DB sentinel does not

apps/webapp/app/runEngine/services/batchTrigger.server.ts

@@ -18,6 +18,7 @@ import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
 import {
   downloadPacketFromObjectStore,
@@ -588,6 +589,9 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
           parentRunId,
           resumeParentOnCompletion,
           batch: { id: batch.id, index: workingIndex },
+          // Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
+          // co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
+          runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
           options: item.options as Record<string, unknown>,
           traceContext: options?.traceContext as Record<string, unknown> | undefined,
           spanParentAsLink: options?.spanParentAsLink,
@@ -677,6 +681,12 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
 
     const triggerTaskService = new TriggerTaskService();
 
+    // Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
+    // an org's mint flag flipping between batch creation and this (possibly much later,
+    // worker-processed) item never splits the item from its BatchTaskRun row. See
+    // mintAnchoredRunFriendlyId.server.ts.
+    const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
+
     const result = await triggerTaskService.call(
       item.task,
       environment,
@@ -695,6 +705,7 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
         spanParentAsLink: options?.spanParentAsLink,
         batchId: batch.id,
         batchIndex: currentIndex,
+        runFriendlyId,
         realtimeStreamsVersion: options?.realtimeStreamsVersion,
         triggerSource: options?.triggerSource ?? "api",
         triggerAction: options?.triggerAction ?? "trigger",

apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it, vi } from "vitest";
+
+// Empty singletons satisfy the module-level wiring imports; the mint method under test is driven
+// directly via (service as any) and never touches the DB (same boundary as triggerTask.server.test.ts).
+vi.mock("~/db.server", () => ({
+  prisma: {},
+  $replica: {},
+  runOpsNewPrisma: {},
+  runOpsLegacyPrisma: {},
+  runOpsNewReplica: {},
+  runOpsLegacyReplica: {},
+}));
+vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
+
+import { classifyKind, generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
+import { TriggerFailedTaskService } from "./triggerFailedTask.server";
+
+function buildService() {
+  return new TriggerFailedTaskService({ prisma: {} as any, engine: {} as any });
+}
+
+describe("TriggerFailedTaskService.mintFailedRunFriendlyId", () => {
+  it("returns the caller-supplied runFriendlyId verbatim (override wins over any mint)", async () => {
+    const override = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      runFriendlyId: override,
+    });
+    expect(minted).toBe(override);
+  });
+
+  it("without an override, still inherits a run-ops (NEW) parent by id-shape", async () => {
+    const parentRunFriendlyId = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      parentRunFriendlyId,
+    });
+    expect(classifyKind(minted)).toBe("runOpsId");
+  });
+});

apps/webapp/app/runEngine/services/triggerFailedTask.server.ts

@@ -43,6 +43,9 @@ export type TriggerFailedTaskRequest = {
   spanParentAsLink?: boolean;
 
   errorCode?: TaskRunErrorCodes;
+
+  /** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
+  runFriendlyId?: string;
 };
 
 /**
@@ -86,14 +89,20 @@ export class TriggerFailedTaskService {
 
   // Mint a failed run's friendlyId. The id-kind decides which store the run is
   // born in (cuid → legacy store, run-ops id → new store); the whole subgraph of a
-  // run must agree. Root failed runs mint by the environment's setting; child
-  // failed runs inherit the parent's current store so they never split.
+  // run must agree. A caller-supplied runFriendlyId (batch-anchored id) wins verbatim;
+  // otherwise root failed runs mint by the environment's setting and child failed runs
+  // inherit the parent's current store so they never split.
   private async mintFailedRunFriendlyId(args: {
     organizationId: string;
     environmentId: string;
     orgFeatureFlags?: unknown;
     parentRunFriendlyId?: string;
+    runFriendlyId?: string;
   }): Promise<string> {
+    if (args.runFriendlyId) {
+      return args.runFriendlyId;
+    }
+
     const mintKind = args.parentRunFriendlyId
       ? resolveInheritedMintKind(args.parentRunFriendlyId)
       : await resolveRunIdMintKind({
@@ -125,6 +134,7 @@ export class TriggerFailedTaskService {
         environmentId: request.environment.id,
         orgFeatureFlags: request.environment.organization.featureFlags,
         parentRunFriendlyId: request.parentRunId,
+        runFriendlyId: request.runFriendlyId,
       });
       mintedFriendlyId = failedRunFriendlyId;
 
@@ -321,6 +331,8 @@ export class TriggerFailedTaskService {
     resumeParentOnCompletion?: boolean;
     batch?: { id: string; index: number };
     errorCode?: TaskRunErrorCodes;
+    /** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
+    runFriendlyId?: string;
   }): Promise<string | null> {
     // Held for the catch's log line; the in-try `const` is what consumers use.
     let mintedFriendlyId: string | undefined;
@@ -335,6 +347,7 @@ export class TriggerFailedTaskService {
         // single replica lookup by organizationId only when there is no parent.
         orgFeatureFlags: undefined,
         parentRunFriendlyId: opts.parentRunId,
+        runFriendlyId: opts.runFriendlyId,
       });
       mintedFriendlyId = failedRunFriendlyId;
 

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -14,7 +14,6 @@ import {
   TriggerTraceContext,
 } from "@trigger.dev/core/v3";
 import {
-  generateRunOpsId,
   parseTraceparent,
   RunId,
   serializeTraceparent,
@@ -28,6 +27,7 @@ import { handleMetadataPacket } from "~/utils/packets";
 import { startSpan } from "~/v3/tracing.server";
 import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
 import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
+import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import type {
   TriggerTaskServiceOptions,
   TriggerTaskServiceResult,
@@ -153,9 +153,7 @@ export class RunEngineTriggerTaskService {
           orgFeatureFlags: environment.organization.featureFlags,
         });
 
-    return mintKind === "runOpsId"
-      ? RunId.toFriendlyId(generateRunOpsId(region))
-      : RunId.generate().friendlyId;
+    return mintFriendlyIdForKind(mintKind, region);
   }
 
   public async call({

apps/webapp/app/v3/runEngineHandlers.server.ts

@@ -28,6 +28,7 @@ import { getEventRepositoryForStore, recordRunDebugLog } from "./eventRepository
 import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server";
 import { engine } from "./runEngine.server";
 import { runStore } from "./runStore.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
 import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
 import {
@@ -808,6 +809,14 @@ export function setupBatchQueueCallbacks() {
           },
         },
         async (span) => {
+          // Anchor every item mint on the BATCH's friendlyId so a mid-batch mint-flag flip
+          // can't split an item (or pre-failed item) from its BatchTaskRun row.
+          const mintItemRunFriendlyId = () =>
+            mintAnchoredRunFriendlyId(
+              friendlyId,
+              (item.options as { region?: string } | undefined)?.region
+            );
+
           const triggerFailedTaskService = new TriggerFailedTaskService({
             prisma,
             engine,
@@ -837,6 +846,7 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                runFriendlyId: mintItemRunFriendlyId(),
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,
               });
@@ -874,6 +884,8 @@ export function setupBatchQueueCallbacks() {
             // Normalize payload - for application/store (R2 paths), this passes through as-is
             const payload = normalizePayload(item.payload, item.payloadType);
 
+            const runFriendlyId = mintItemRunFriendlyId();
+
             const result = await triggerTaskService.call(
               item.task,
               environment,
@@ -893,6 +905,7 @@ export function setupBatchQueueCallbacks() {
                 spanParentAsLink: meta.spanParentAsLink,
                 batchId,
                 batchIndex: itemIndex,
+                runFriendlyId,
                 realtimeStreamsVersion: meta.realtimeStreamsVersion,
                 planType: meta.planType,
                 triggerSource: meta.parentRunId ? "sdk" : (meta.triggerSource ?? "api"),
@@ -929,6 +942,7 @@ export function setupBatchQueueCallbacks() {
                   parentRunId: meta.parentRunId,
                   resumeParentOnCompletion: meta.resumeParentOnCompletion,
                   batch: { id: batchId, index: itemIndex },
+                  runFriendlyId: mintItemRunFriendlyId(),
                   options: item.options as Record<string, unknown>,
                   traceContext: meta.traceContext as Record<string, unknown> | undefined,
                   spanParentAsLink: meta.spanParentAsLink,
@@ -1009,6 +1023,7 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                runFriendlyId: mintItemRunFriendlyId(),
                 options: item.options as Record<string, unknown>,
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,

apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts

@@ -0,0 +1,31 @@
+import {
+  BatchId,
+  classifyKind,
+  generateRunOpsId,
+  parseRunId,
+  REGION_CODES,
+} from "@trigger.dev/core/v3/isomorphic";
+import { describe, expect, it } from "vitest";
+import { mintAnchoredRunFriendlyId } from "./mintAnchoredRunFriendlyId.server";
+
+describe("mintAnchoredRunFriendlyId", () => {
+  it("a run-ops (NEW) batch anchor yields a run-ops (NEW) item friendlyId", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("runOpsId");
+  });
+
+  it("a cuid (LEGACY) batch anchor yields a cuid (LEGACY) item friendlyId", () => {
+    const batchFriendlyId = BatchId.generate().friendlyId;
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("cuid");
+  });
+
+  it("stamps the requested region char into a run-ops id", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId, "us-east-1");
+    const parsed = parseRunId(itemFriendlyId);
+    expect(parsed.format).toBe("b32hex");
+    expect(parsed.format === "b32hex" && parsed.region).toBe(REGION_CODES["us-east-1"]);
+  });
+});

apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts

@@ -0,0 +1,15 @@
+import { generateRunOpsId, RunId, type ResidencyKind } from "@trigger.dev/core/v3/isomorphic";
+import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server";
+
+// Shared id-generation branch for every run-mint path: "runOpsId" -> NEW store, "cuid" -> LEGACY.
+export function mintFriendlyIdForKind(mintKind: ResidencyKind, region?: string): string {
+  return mintKind === "runOpsId"
+    ? RunId.toFriendlyId(generateRunOpsId(region))
+    : RunId.generate().friendlyId;
+}
+
+// Anchor a batch item's mint on the BATCH's friendlyId (id-shape, zero I/O), never the per-org
+// flag, so the item and its BatchTaskRun stay co-resident across a mid-batch flag flip.
+export function mintAnchoredRunFriendlyId(batchFriendlyId: string, region?: string): string {
+  return mintFriendlyIdForKind(resolveInheritedMintKind(batchFriendlyId), region);
+}

apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts

@@ -6,9 +6,21 @@ export function computeRunOpsSplitReadEnabled(args: {
   controlPlaneReplica: unknown;
   hasNewUrl: boolean;
   hasLegacyUrl: boolean;
+  logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void };
 }): boolean {
   const newIsDistinctDedicatedClient =
     args.newReplica !== args.controlPlaneWriter && args.newReplica !== args.controlPlaneReplica;
 
-  return newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl;
+  const enabled = newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl;
+
+  // Configured for split but the identity check failed: fan-out is being silently disabled.
+  if (!newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl) {
+    args.logger?.warn(
+      "run-ops split read fan-out is configured (RUN_OPS_DATABASE_URL and " +
+        "RUN_OPS_LEGACY_DATABASE_URL are both set) but the NEW client is not a distinct " +
+        "instance from the control-plane client; read fan-out is silently disabled."
+    );
+  }
+
+  return enabled;
 }