feat: stand up sandbox cluster (Kustomize + Gateway API + push daemon)

.github/workflows/ci.yml

@@ -21,6 +21,11 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      # @cfp/shared's exports map points at dist/. Build it first so type-check,
+      # lint, and test can resolve `@cfp/shared` / `@cfp/shared/schemas`.
+      - name: Build @cfp/shared
+        run: npm run -w packages/shared build
+
       - name: Type check
         run: npm run type-check
 

Dockerfile

@@ -47,18 +47,17 @@ WORKDIR /app
 
 RUN apk add --no-cache git python3 make g++
 
+# npm workspaces hoists every dep to the root node_modules; the per-workspace
+# node_modules dirs don't exist at this scale. Copy only the root.
 COPY --from=deps /app/node_modules ./node_modules
-COPY --from=deps /app/apps/api/node_modules ./apps/api/node_modules
-COPY --from=deps /app/apps/web/node_modules ./apps/web/node_modules
-COPY --from=deps /app/packages/shared/node_modules ./packages/shared/node_modules
 
 COPY tsconfig.base.json package.json package-lock.json ./
 COPY apps ./apps
 COPY packages ./packages
 
-# Build both workspaces. Web is built first so api/dist references work; the
-# workspace `build` script handles order via `--if-present`.
-RUN npm run build --workspaces --if-present
+# Build in dependency order: shared first (api + web both import from it),
+# then api + web. The root `build` script enforces this ordering.
+RUN npm run build
 
 # Drop devDependencies from node_modules to shrink the runtime image. We still
 # need workspace-local node_modules (better-sqlite3 native binding lives there).
@@ -77,16 +76,14 @@ RUN apk add --no-cache git ca-certificates tini openssh-client
 
 WORKDIR /app
 
-# Copy built artifacts + pruned node_modules.
+# Copy built artifacts + pruned node_modules (hoisted at the root).
 COPY --from=build /app/package.json /app/package-lock.json ./
 COPY --from=build /app/node_modules ./node_modules
 COPY --from=build /app/apps/api/package.json ./apps/api/
 COPY --from=build /app/apps/api/dist ./apps/api/dist
-COPY --from=build /app/apps/api/node_modules ./apps/api/node_modules
 COPY --from=build /app/apps/web/dist ./apps/web/dist
 COPY --from=build /app/packages/shared/package.json ./packages/shared/
 COPY --from=build /app/packages/shared/dist ./packages/shared/dist
-COPY --from=build /app/packages/shared/node_modules ./packages/shared/node_modules
 
 # Entrypoint script handles data-repo init/refresh before exec'ing node.
 COPY deploy/docker/entrypoint.sh /usr/local/bin/entrypoint.sh

apps/api/scripts/cutover-dry-run.ts

@@ -327,7 +327,7 @@ export async function runDryRun(opts: DryRunOptions): Promise<DryRunReport> {
 
   let smokeChecks: SmokeCheckResult[] = [];
   if (opts.target) {
-    const publicStore = await openPublicStore(opts.dataRepo);
+    const { store: publicStore } = await openPublicStore(opts.dataRepo);
     const people = await publicStore.people.queryAll();
     const projects = await publicStore.projects.queryAll();
     const liveProjects = projects.filter((p) => !p.deletedAt);

apps/api/scripts/cutover-mailout.ts

@@ -22,7 +22,7 @@
 import { writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
-import { openPublicStore } from '../src/store/public.js';
+import { openPublicStore, type PublicStore } from '../src/store/public.js';
 import {
   FilesystemPrivateStore,
   S3PrivateStore,
@@ -50,7 +50,7 @@ export interface MailoutReport {
 }
 
 export interface MailoutOptions {
-  readonly publicStore: Awaited<ReturnType<typeof openPublicStore>>;
+  readonly publicStore: PublicStore;
   readonly privateStore: PrivateStore;
   readonly mode: 'dry-run' | 'send';
   readonly from?: string;
@@ -64,7 +64,7 @@ export interface MailoutOptions {
 // ---------------------------------------------------------------------------
 
 export async function collectRecipients(
-  publicStore: Awaited<ReturnType<typeof openPublicStore>>,
+  publicStore: PublicStore,
   privateStore: PrivateStore,
 ): Promise<{ recipients: MailoutRecipient[]; skipped: Array<{ personId: string; reason: string }> }> {
   const people = await publicStore.people.queryAll();
@@ -289,7 +289,7 @@ async function main(): Promise<void> {
     process.exit(2);
   }
 
-  const publicStore = await openPublicStore(requireEnv('CFP_DATA_REPO_PATH'));
+  const { store: publicStore } = await openPublicStore(requireEnv('CFP_DATA_REPO_PATH'));
   const privateStore = buildPrivateStore();
   await privateStore.load();
 

apps/api/scripts/reconcile.ts

@@ -41,7 +41,7 @@ import {
   PrivateProfileSchema,
   type PrivateProfile,
 } from '@cfp/shared/schemas';
-import { openPublicStore } from '../src/store/public.js';
+import { openPublicStore, type PublicStore } from '../src/store/public.js';
 import {
   FilesystemPrivateStore,
   S3PrivateStore,
@@ -118,7 +118,7 @@ function buildPrivateStore(): PrivateStore {
 // ---------------------------------------------------------------------------
 
 export interface ReconcileOptions {
-  readonly publicStore: Awaited<ReturnType<typeof openPublicStore>>;
+  readonly publicStore: PublicStore;
   readonly privateStore: PrivateStore;
   readonly fix?: boolean;
   readonly now?: string;
@@ -305,7 +305,7 @@ async function main(): Promise<void> {
   const args = parseArgs(process.argv.slice(2));
 
   const repoPath = requireEnv('CFP_DATA_REPO_PATH');
-  const publicStore = await openPublicStore(repoPath);
+  const { store: publicStore } = await openPublicStore(repoPath);
   const privateStore = buildPrivateStore();
   await privateStore.load();
 

apps/api/src/app.ts

@@ -29,6 +29,7 @@ import { envJsonSchema, type Env } from './env.js';
 import { mapError } from './lib/errors.js';
 import traceIdPlugin from './plugins/trace-id.js';
 import storePlugin from './plugins/store.js';
+import pushDaemonPlugin from './plugins/push-daemon.js';
 import servicesPlugin from './plugins/services.js';
 import rateLimitPlugin from './plugins/rate-limit.js';
 import idempotencyPlugin from './plugins/idempotency.js';
@@ -110,6 +111,9 @@ export async function buildApp(opts: BuildAppOptions = {}): Promise<FastifyInsta
   // ----- 6. Store (boots gitsheets + private-store) -----
   await fastify.register(storePlugin);
 
+  // ----- 6a. Push daemon (pushes public-store commits to CFP_DATA_REMOTE) -----
+  await fastify.register(pushDaemonPlugin);
+
   // ----- 6b. Services (loads in-memory state + FTS, boots after store) -----
   await fastify.register(servicesPlugin);
 

apps/api/src/auth/middleware.ts

@@ -104,11 +104,15 @@ async function sessionMiddlewarePlugin(fastify: FastifyInstance): Promise<void>
         return;
       }
 
-      // Look up person from public store
-      const person = await fastify.store.public.people.queryFirst({ id: claims.sub } as Record<string, unknown>);
+      // Look up person from the in-memory state map (keyed by id). Other
+      // routes use the same path (`fastify.inMemoryState.people.get(personId)`)
+      // for id→Person resolution; this is the canonical fast index. The
+      // sheet-level `queryFirst({ id })` previously used here doesn't reflect
+      // in-process writes between commit and the next refresh.
+      const person = fastify.inMemoryState.people.get(claims.sub) ?? null;
 
       request.session = {
-        person: person ?? null,
+        person,
         accountLevel: claims.accountLevel,
         personId: claims.sub,
         jti: claims.jti,

apps/api/src/env.ts

@@ -15,6 +15,8 @@ export const EnvSchema = z.object({
   CFP_DATA_REPO_PATH: z.string(),
   /** Git remote URL to push public data commits to (optional in dev). */
   CFP_DATA_REMOTE: z.string().optional(),
+  /** Branch the push daemon pushes to. Defaults to the repo's current HEAD. */
+  CFP_DATA_BRANCH: z.string().optional(),
   /** Which private-storage backend to use. */
   STORAGE_BACKEND: z.enum(['s3', 'filesystem']),
   /** Filesystem backend: absolute path to the private-storage directory. */
@@ -70,6 +72,7 @@ export const envJsonSchema = {
     },
     CFP_DATA_REPO_PATH: { type: 'string' },
     CFP_DATA_REMOTE: { type: 'string' },
+    CFP_DATA_BRANCH: { type: 'string' },
     STORAGE_BACKEND: { type: 'string', enum: ['s3', 'filesystem'] },
     CFP_PRIVATE_STORAGE_PATH: { type: 'string' },
     S3_ENDPOINT: { type: 'string' },

apps/api/src/plugins/push-daemon.ts

@@ -0,0 +1,87 @@
+/**
+ * Push-daemon plugin.
+ *
+ * Starts gitsheets' async push daemon against `origin` for the public data
+ * repo when `CFP_DATA_REMOTE` is set. The daemon pushes new commits as soon
+ * as they're notified (via Repository.transact), with exponential backoff
+ * retries on transient failures. Non-fast-forward rejections are logged
+ * loudly without retry (terminal — operator must reconcile).
+ *
+ * See specs/behaviors/storage.md#push-sync and GitHub issue #37.
+ *
+ * Skipped entirely when `CFP_DATA_REMOTE` is unset (typical for local dev
+ * where developers iterate against a sibling working tree with no remote).
+ *
+ * Depends on `store` (which sets up `fastify.publicRepo`).
+ */
+import type { FastifyInstance } from 'fastify';
+import fp from 'fastify-plugin';
+import type { PushDaemon } from 'gitsheets';
+
+declare module 'fastify' {
+  interface FastifyInstance {
+    pushDaemon: PushDaemon | null;
+  }
+}
+
+async function pushDaemonPlugin(fastify: FastifyInstance): Promise<void> {
+  if (!fastify.config.CFP_DATA_REMOTE) {
+    fastify.log.info(
+      'push-daemon disabled: CFP_DATA_REMOTE unset (local commits stay local)',
+    );
+    fastify.decorate('pushDaemon', null);
+    return;
+  }
+
+  const daemon = await fastify.publicRepo.startPushDaemon({
+    remote: 'origin',
+    branch: fastify.config.CFP_DATA_BRANCH,
+    backoff: 'exponential',
+  });
+
+  daemon.on('push', ({ commit, durationMs }: { commit: string; durationMs: number }) => {
+    fastify.log.info({ commit, durationMs }, 'pushed commit to origin');
+  });
+  daemon.on('retry', ({ attempt, nextDelayMs }: { attempt: number; nextDelayMs: number }) => {
+    fastify.log.info({ attempt, nextDelayMs }, 'push-daemon retrying');
+  });
+  daemon.on(
+    'error',
+    ({
+      err,
+      attempt,
+      reason,
+    }: {
+      err: unknown;
+      attempt: number;
+      reason: 'non-fast-forward' | 'unknown';
+    }) => {
+      if (reason === 'non-fast-forward') {
+        fastify.log.error(
+          { err: String(err), attempt },
+          'push rejected non-fast-forward — manual reconciliation required',
+        );
+      } else {
+        fastify.log.warn({ err: String(err), attempt, reason }, 'push attempt failed');
+      }
+    },
+  );
+
+  fastify.decorate('pushDaemon', daemon);
+
+  fastify.addHook('onClose', async () => {
+    fastify.log.info('stopping push-daemon');
+    await daemon.stop();
+  });
+
+  fastify.log.info(
+    { remote: 'origin', branch: fastify.config.CFP_DATA_BRANCH ?? 'HEAD' },
+    'push-daemon started',
+  );
+}
+
+export default fp(pushDaemonPlugin, {
+  name: 'push-daemon',
+  fastify: '5.x',
+  dependencies: ['store'],
+});

apps/api/src/plugins/store.ts

@@ -9,17 +9,20 @@
  */
 import type { FastifyInstance } from 'fastify';
 import fp from 'fastify-plugin';
+import type { Repository } from 'gitsheets';
 import { bootStores } from '../store/boot.js';
 import type { Store } from '../store/store.js';
 
 declare module 'fastify' {
   interface FastifyInstance {
     store: Store;
+    /** Underlying gitsheets repo for the public data store — used by the push-daemon plugin. */
+    publicRepo: Repository;
   }
 }
 
 async function storePlugin(fastify: FastifyInstance): Promise<void> {
-  const store = await bootStores({
+  const { store, publicRepo } = await bootStores({
     CFP_DATA_REPO_PATH: fastify.config.CFP_DATA_REPO_PATH,
     STORAGE_BACKEND: fastify.config.STORAGE_BACKEND,
     CFP_PRIVATE_STORAGE_PATH: fastify.config.CFP_PRIVATE_STORAGE_PATH,
@@ -31,6 +34,7 @@ async function storePlugin(fastify: FastifyInstance): Promise<void> {
   });
 
   fastify.decorate('store', store);
+  fastify.decorate('publicRepo', publicRepo);
 }
 
 export default fp(storePlugin, {

apps/api/src/routes/auth.ts

@@ -288,7 +288,7 @@ export async function authRoutes(fastify: FastifyInstance): Promise<void> {
         throw new UnauthenticatedError('Refresh token revoked', 'refresh_token_revoked');
       }
 
-      const person = await fastify.store.public.people.queryFirst({ id: claims.sub });
+      const person = fastify.inMemoryState.people.get(claims.sub);
       if (!person) {
         throw new UnauthenticatedError('Person not found', 'refresh_token_revoked');
       }

apps/api/src/routes/saml.ts

@@ -227,9 +227,7 @@ async function loadPersonAndProfile(
   fastify: FastifyInstance,
   personId: string,
 ): Promise<{ person: Person; profile: PrivateProfile }> {
-  const person = (await fastify.store.public.people.queryFirst({ id: personId })) as
-    | Person
-    | undefined;
+  const person = fastify.inMemoryState.people.get(personId);
   if (!person) {
     throw new UnauthenticatedError('Person not found', 'unauthenticated');
   }

apps/api/src/store/boot.ts

@@ -1,3 +1,4 @@
+import type { Repository } from 'gitsheets';
 import { FilesystemPrivateStore } from './private/filesystem.js';
 import { S3PrivateStore } from './private/s3.js';
 import { openPublicStore } from './public.js';
@@ -21,7 +22,9 @@ export interface Env {
 }
 
 /**
- * Boot both stores and return a combined Store.
+ * Boot both stores and return a combined Store + the underlying public-repo
+ * handle. The repo handle is consumed by the push-daemon plugin to push
+ * commits to origin; everything else only needs `store`.
  *
  * Fails loudly (throws) if either store is unreachable. The API must not
  * serve traffic until this resolves — private profiles are required for login.
@@ -31,8 +34,8 @@ export interface Env {
  *   2. Private store data
  *   3. (FTS index is built by the caller from the loaded public data)
  */
-export async function bootStores(env: Env): Promise<Store> {
-  const publicStore = await openPublicStore(env.CFP_DATA_REPO_PATH).catch((err) => {
+export async function bootStores(env: Env): Promise<{ store: Store; publicRepo: Repository }> {
+  const { store: publicStore, repo: publicRepo } = await openPublicStore(env.CFP_DATA_REPO_PATH).catch((err) => {
     throw new Error(`Failed to open public gitsheets store at ${env.CFP_DATA_REPO_PATH}: ${String(err)}`, { cause: err });
   });
 
@@ -46,7 +49,7 @@ export async function bootStores(env: Env): Promise<Store> {
     throw new Error(`Failed to load private store (${env.STORAGE_BACKEND}): ${String(err)}`, { cause: err });
   });
 
-  return new Store(publicStore, privateStore);
+  return { store: new Store(publicStore, privateStore), publicRepo };
 }
 
 function buildPrivateStore(env: Env): FilesystemPrivateStore | S3PrivateStore {

apps/api/src/store/public.ts

@@ -1,5 +1,5 @@
 import { openRepo, openStore } from 'gitsheets';
-import type { StandardSchemaV1, Store, StoreTx, ValidatorMap } from 'gitsheets';
+import type { Repository, StandardSchemaV1, Store, StoreTx, ValidatorMap } from 'gitsheets';
 import {
   HelpWantedInterestExpressionSchema,
   HelpWantedRoleSchema,
@@ -63,8 +63,13 @@ export type PublicStoreTx = StoreTx<PublicValidators>;
  * Reads `.gitsheets/<sheet>.toml` for each declared sheet in `repoPath`.
  * In-memory secondary indices are built by the caller (boot.ts) after this
  * returns, since they require iterating over all records.
+ *
+ * Returns both the typed store and the underlying Repository handle — the
+ * latter is needed by the push-daemon plugin to push commits to origin.
  */
-export async function openPublicStore(repoPath: string): Promise<PublicStore> {
+export async function openPublicStore(
+  repoPath: string,
+): Promise<{ store: PublicStore; repo: Repository }> {
   const repo = await openRepo({ gitDir: `${repoPath}/.git`, workTree: repoPath });
   repo.requireExplicitTransactions();
 
@@ -82,5 +87,6 @@ export async function openPublicStore(repoPath: string): Promise<PublicStore> {
     revocations: asValidator<Revocation>(RevocationSchema),
   };
 
-  return openStore(repo, { validators }) as Promise<PublicStore>;
+  const store = (await openStore(repo, { validators })) as PublicStore;
+  return { store, repo };
 }