fix(sandboxes): rebuild a hash adopted while its image was being deleted

Greptile's third pass on this path, and a case the previous two did not cover:
the adopter starting a *fresh build* rather than inheriting a ready row.

Claiming removes the registry row, so between that and the provider delete
finishing, a workspace can declare the same package list, get a new row, and start
a build under the same content-derived imageRef — which the in-flight delete then
removes.

The window itself is inherent. The registry row and the provider template are two
systems with no shared transaction, so it can be narrowed but not closed. A Redis
lock would not close it either: acquireLock returns true when Redis is absent, so
it cannot be a correctness guarantee for self-hosted. Holding a Postgres advisory
lock would, but only by pinning a pooled connection for the length of a provider
call, which is a worse trade.

What was avoidable is the adopter finding out the slow way. Its row is new and
healthy-looking, so nothing noticed: resolution only repairs a row that is missing
or failed, and a failed one waits out the retry cooldown first. The release path
now re-checks after the delete and re-enqueues, so the rebuild starts immediately
instead of one failed run plus a cooldown later. A build already in flight is left
to the conflict guard, since it may still outlive the delete.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: icecrasher321

apps/sim/lib/execution/remote-sandbox/image-registry.test.ts

@@ -72,6 +72,7 @@ import {
 const READY_IMAGE = {
   id: 'img-1',
   status: 'ready',
+  spec: { language: 'python', dependencies: ['pandas'] },
   imageRef: 'sim-sbx-abc',
   buildId: 'build-1',
   providerImageId: 'tmpl-1',
@@ -93,6 +94,25 @@ function predicateText(predicate: unknown): string {
   return String(predicate)
 }
 
+/**
+ * True once a build upsert has been issued. Distinguished from the restore path by
+ * the conflict clause: a rebuild is `onConflictDoUpdate`, a restore is
+ * `onConflictDoNothing`.
+ */
+function captureUpsert(): () => boolean {
+  let seen = false
+  mockInsert.mockReturnValue({
+    values: () => ({
+      onConflictDoUpdate: () => {
+        seen = true
+        return { returning: () => Promise.resolve([]) }
+      },
+      onConflictDoNothing: () => Promise.resolve(),
+    }),
+  })
+  return () => seen
+}
+
 /** Captures the conditional-delete predicate and what the claim resolves to. */
 function stubClaim(rows: unknown[]): () => unknown {
   let captured: unknown
@@ -176,6 +196,36 @@ describe('releaseSandboxImage', () => {
     expect(mockInsert).toHaveBeenCalledTimes(1)
   })
 
+  /**
+   * The adopter's row is new and healthy-looking, so nothing else would notice the
+   * template went out from under it — resolution only repairs a missing or
+   * `failed` row, and a `failed` one waits out the cooldown first.
+   */
+  it('rebuilds a hash re-adopted while the provider delete was in flight', async () => {
+    stubClaim([READY_IMAGE])
+    mockSelect.mockReturnValue({
+      from: () => ({ where: () => ({ limit: () => Promise.resolve([{ id: 'img-new' }]) }) }),
+    })
+    const enqueued = captureUpsert()
+
+    await releaseSandboxImage('hash-1')
+
+    expect(mockDeleteImage).toHaveBeenCalledTimes(1)
+    expect(enqueued()).toBe(true)
+  })
+
+  it('does not rebuild when nothing re-adopted the hash', async () => {
+    stubClaim([READY_IMAGE])
+    mockSelect.mockReturnValue({
+      from: () => ({ where: () => ({ limit: () => Promise.resolve([]) }) }),
+    })
+    const enqueued = captureUpsert()
+
+    await releaseSandboxImage('hash-1')
+
+    expect(enqueued()).toBe(false)
+  })
+
   it('skips the provider when the claimed row never had an image', async () => {
     stubClaim([{ ...READY_IMAGE, imageRef: null }])
 
@@ -195,9 +245,15 @@ describe('releaseSandboxImage', () => {
 describe('cleanupSandboxImages', () => {
   const CANDIDATE = { id: 'img-1', specHash: 'hash-1', imageRef: 'sim-sbx-abc' }
 
-  /** One candidate row from the sweep's nomination query. */
+  /**
+   * Nomination query only — later selects (the re-adoption check) resolve empty, so
+   * a candidate is not mistaken for its own adopter.
+   */
   function stubCandidates(rows: unknown[]) {
     mockSelect.mockReturnValue({
+      from: () => ({ where: () => ({ limit: () => Promise.resolve([]) }) }),
+    })
+    mockSelect.mockReturnValueOnce({
       from: () => ({ where: () => ({ limit: () => Promise.resolve(rows) }) }),
     })
   }

apps/sim/lib/execution/remote-sandbox/image-registry.ts

@@ -329,6 +329,38 @@ export async function releaseSandboxImage(specHash: string): Promise<void> {
 
 type ImageClaimOutcome = 'released' | 'skipped' | 'failed'
 
+/**
+ * Rebuilds a hash that was adopted while its image was being deleted.
+ *
+ * Claiming removes the row, so between that and the provider call finishing a
+ * workspace can declare the same package list, get a fresh row, and start a build
+ * under the same content-derived `imageRef` — which the in-flight delete then
+ * removes. The window is inherent: the registry row and the provider template are
+ * two systems with no shared transaction, so it can be made small but not zero.
+ *
+ * What is avoidable is the adopter finding out the slow way. Its row is new and
+ * healthy-looking, so nothing else would notice: resolution only repairs a row
+ * that is missing or `failed`, and a `failed` one waits out
+ * {@link FAILED_BUILD_RETRY_COOLDOWN_MS} first. Re-enqueueing here converts that
+ * into a rebuild starting immediately. A build already in flight is left alone by
+ * the conflict guard, since it may still outlive the delete.
+ */
+async function rebuildIfReadopted(
+  providerId: string,
+  specHash: string,
+  spec: SandboxSpec
+): Promise<void> {
+  const [readopted] = await db
+    .select({ id: sandboxImage.id })
+    .from(sandboxImage)
+    .where(and(eq(sandboxImage.provider, providerId), eq(sandboxImage.specHash, specHash)))
+    .limit(1)
+  if (!readopted) return
+
+  logger.warn('Sandbox image was re-adopted mid-delete; rebuilding it now', { specHash })
+  await ensureSandboxImage(spec, specHash)
+}
+
 /**
  * Takes ownership of a spec hash's registry row and deletes the image behind it.
  *
@@ -383,6 +415,7 @@ async function claimAndDeleteImage(
       buildId: claimed.buildId ?? '',
       providerImageId: claimed.providerImageId ?? undefined,
     })
+    await rebuildIfReadopted(providerId, specHash, claimed.spec as SandboxSpec)
     return 'released'
   } catch (error) {
     await db