feat(laddr-import): one-shot migration script from laddr mysqldump

apps/api/package.json

@@ -11,7 +11,8 @@
     "type-check": "tsc -p tsconfig.json --noEmit",
     "test": "vitest run",
     "script:scrub-data": "tsx scripts/scrub-data.ts",
-    "script:setup-dev-data": "tsx scripts/setup-dev-data.ts"
+    "script:setup-dev-data": "tsx scripts/setup-dev-data.ts",
+    "script:import-laddr": "tsx scripts/import-laddr.ts"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1048.0",

apps/api/scripts/fixtures/laddr-fixture.sql

@@ -0,0 +1,111 @@
+-- Synthetic laddr mysqldump fixture for import-laddr tests.
+-- Mirrors the shape (CREATE TABLE then INSERT) of real laddr dumps.
+
+CREATE TABLE `people` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `Username` varchar(255) NOT NULL,
+  `FirstName` varchar(255) DEFAULT NULL,
+  `LastName` varchar(255) DEFAULT NULL,
+  `FullName` varchar(255) DEFAULT NULL,
+  `Email` varchar(255) DEFAULT NULL,
+  `Password` varchar(255) DEFAULT NULL,
+  `About` text DEFAULT NULL,
+  `AccountLevel` varchar(64) DEFAULT 'User',
+  `Created` datetime DEFAULT NULL,
+  `Modified` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `people` VALUES (1,'jane-doe','Jane','Doe','Jane Doe','jane@example.com','$2y$10$abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQ','Civic technologist.','Administrator','2020-01-15 18:42:00','2024-05-01 09:00:00');
+INSERT INTO `people` VALUES (2,'bobsmith','Bob','Smith',NULL,'bob@example.org','$2y$10$xyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyz','I like buses.','User','2021-06-20 12:00:00','2021-06-20 12:00:00'),(3,'Weird Name!','Carol','Singh','Carol Singh','carol@example.net',NULL,NULL,'User','2022-03-01 00:00:00','2022-03-01 00:00:00');
+INSERT INTO `people` VALUES (4,'no-email','Dee','Park','Dee Park',NULL,NULL,NULL,'User','2023-01-01 00:00:00','2023-01-01 00:00:00');
+
+CREATE TABLE `projects` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `Handle` varchar(255) NOT NULL,
+  `Title` varchar(255) NOT NULL,
+  `Summary` varchar(280) DEFAULT NULL,
+  `README` text DEFAULT NULL,
+  `Stage` varchar(64) DEFAULT 'Commenting',
+  `MaintainerID` int(11) DEFAULT NULL,
+  `UsersUrl` varchar(255) DEFAULT NULL,
+  `DevelopersUrl` varchar(255) DEFAULT NULL,
+  `ChatChannel` varchar(64) DEFAULT NULL,
+  `Created` datetime DEFAULT NULL,
+  `Modified` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `projects` VALUES (10,'squadquest','SquadQuest','Realtime events.','## Overview\n\nSquadQuest is a civic app.','Testing',1,'https://squadquest.app','https://github.com/example/squadquest','squadquest','2020-02-01 00:00:00','2024-04-15 00:00:00');
+INSERT INTO `projects` VALUES (11,'transit-tools','Transit Tools','Better SEPTA info.',NULL,'Prototyping',2,NULL,'https://github.com/example/transit-tools','transit','2021-01-01 00:00:00','2021-01-01 00:00:00');
+
+CREATE TABLE `project_members` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `ProjectID` int(11) NOT NULL,
+  `PersonID` int(11) NOT NULL,
+  `Role` varchar(255) DEFAULT NULL,
+  `Joined` datetime DEFAULT NULL,
+  `Created` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `project_members` VALUES (100,10,1,'Maintainer','2020-02-01 00:00:00','2020-02-01 00:00:00'),(101,10,2,'Backend Engineer','2020-03-01 00:00:00','2020-03-01 00:00:00'),(102,11,2,'Founder','2021-01-01 00:00:00','2021-01-01 00:00:00');
+
+CREATE TABLE `project_updates` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `ProjectID` int(11) NOT NULL,
+  `AuthorID` int(11) DEFAULT NULL,
+  `Update` text NOT NULL,
+  `Created` datetime DEFAULT NULL,
+  `Modified` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `project_updates` VALUES (200,10,1,'We shipped v1.0!','2024-03-01 00:00:00','2024-03-01 00:00:00');
+INSERT INTO `project_updates` VALUES (201,10,2,'Beta testers wanted.','2024-04-01 00:00:00','2024-04-01 00:00:00'),(202,11,2,'First commit.','2021-01-02 00:00:00','2021-01-02 00:00:00');
+
+CREATE TABLE `project_buzz` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `ProjectID` int(11) NOT NULL,
+  `PostedByID` int(11) DEFAULT NULL,
+  `Headline` varchar(255) NOT NULL,
+  `URL` varchar(500) NOT NULL,
+  `Published` datetime DEFAULT NULL,
+  `Summary` text DEFAULT NULL,
+  `Created` datetime DEFAULT NULL,
+  `Modified` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `project_buzz` VALUES (300,10,1,'The Inquirer praises SquadQuest','https://www.inquirer.com/tech/squadquest','2024-01-15 00:00:00','Great review.','2024-01-15 00:00:00','2024-01-15 00:00:00');
+
+CREATE TABLE `tags` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `Handle` varchar(255) NOT NULL,
+  `Title` varchar(255) NOT NULL,
+  `Created` datetime DEFAULT NULL,
+  `Modified` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `tags` VALUES (500,'tech.flutter','Flutter','2020-01-01 00:00:00','2020-01-01 00:00:00'),(501,'topic.transit','Transit','2020-01-01 00:00:00','2020-01-01 00:00:00'),(502,'event.hackathon','Hackathon','2020-01-01 00:00:00','2020-01-01 00:00:00');
+
+CREATE TABLE `tag_items` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `TagID` int(11) NOT NULL,
+  `ContextClass` varchar(255) NOT NULL,
+  `ContextID` int(11) NOT NULL,
+  `Created` datetime DEFAULT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `tag_items` VALUES (600,500,'Emergence\\\\Models\\\\Project',10,'2020-02-01 00:00:00'),(601,501,'Emergence\\\\Models\\\\Project',11,'2021-01-01 00:00:00'),(602,500,'Emergence\\\\People\\\\Person',1,'2020-02-01 00:00:00');
+
+-- Tables we deliberately skip per specs/deferred.md
+CREATE TABLE `member_checkins` (
+  `ID` int(11) NOT NULL AUTO_INCREMENT,
+  `PersonID` int(11) NOT NULL,
+  PRIMARY KEY (`ID`)
+);
+
+INSERT INTO `member_checkins` VALUES (1000,1);

apps/api/scripts/import-laddr.ts

@@ -0,0 +1,134 @@
+/**
+ * import-laddr.ts — One-shot migration from a laddr mysqldump
+ *
+ * Reads a mysqldump (`--sql`), translates each row to the v1 data model
+ * (Zod-validated against `@cfp/shared/schemas`), and writes records into:
+ *
+ *   - the public gitsheets data repo (`--data-repo`)
+ *   - the private filesystem store (`--private-store`)
+ *
+ * Idempotent on `legacyId`: re-running against the same dump + target
+ * skips rows already present. See specs/behaviors/legacy-id-mapping.md.
+ *
+ * Usage:
+ *   npm run -w apps/api script:import-laddr -- \
+ *     --sql=./scratch/laddr.sql \
+ *     --data-repo=./codeforphilly-data \
+ *     --private-store=./scratch/private-storage \
+ *     [--dry-run] [--verbose] [--limit=N]
+ */
+import { resolve } from 'node:path';
+
+import { FilesystemPrivateStore } from '../src/store/private/filesystem.js';
+import { importLaddr, type ImportReport } from './import-laddr/importer.js';
+
+interface CliArgs {
+  readonly sql: string;
+  readonly dataRepo: string;
+  readonly privateStore: string;
+  readonly dryRun: boolean;
+  readonly verbose: boolean;
+  readonly limit: number | undefined;
+}
+
+function parseArgs(argv: readonly string[]): CliArgs {
+  const opts: Record<string, string | true> = {};
+  for (const a of argv) {
+    if (!a.startsWith('--')) continue;
+    const eq = a.indexOf('=');
+    if (eq === -1) opts[a.slice(2)] = true;
+    else opts[a.slice(2, eq)] = a.slice(eq + 1);
+  }
+  const need = (k: string): string => {
+    const v = opts[k];
+    if (typeof v !== 'string' || !v) {
+      process.stderr.write(`missing --${k}=<path>\n`);
+      process.exit(2);
+    }
+    return v;
+  };
+  const limitRaw = opts['limit'];
+  const limit =
+    typeof limitRaw === 'string' ? Number.parseInt(limitRaw, 10) : undefined;
+
+  return {
+    sql: resolve(need('sql')),
+    dataRepo: resolve(need('data-repo')),
+    privateStore: resolve(need('private-store')),
+    dryRun: opts['dry-run'] === true,
+    verbose: opts['verbose'] === true,
+    limit: Number.isFinite(limit ?? NaN) ? limit : undefined,
+  };
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+
+  const privateStore = new FilesystemPrivateStore({
+    CFP_PRIVATE_STORAGE_PATH: args.privateStore,
+  });
+  await privateStore.load();
+
+  console.log(`[import-laddr] sql=${args.sql}`);
+  console.log(`[import-laddr] data-repo=${args.dataRepo}`);
+  console.log(`[import-laddr] private-store=${args.privateStore}`);
+  console.log(`[import-laddr] dry-run=${args.dryRun} limit=${args.limit ?? 'none'}`);
+
+  const report = await importLaddr({
+    sql: args.sql,
+    dataRepo: args.dataRepo,
+    privateStore,
+    dryRun: args.dryRun,
+    verbose: args.verbose,
+    limit: args.limit,
+  });
+
+  printReport(report, args.dryRun);
+}
+
+function printReport(report: ImportReport, dryRun: boolean): void {
+  const lines: string[] = [];
+  lines.push(`\n=== import-laddr report ===`);
+  lines.push(`runAt:        ${report.runAt}`);
+  lines.push(`sourceSha256: ${report.sourceSha256}`);
+  for (const [sheet, r] of Object.entries(report.entities)) {
+    lines.push(
+      `  ${sheet.padEnd(22)} input=${r.input} imported=${r.imported} skipped=${r.skipped} errors=${r.errors}`,
+    );
+  }
+  lines.push(`warnings: ${report.warnings.length}`);
+  for (const w of report.warnings.slice(0, 25)) lines.push(`  ${w}`);
+  if (report.warnings.length > 25) {
+    lines.push(`  ... (${report.warnings.length - 25} more)`);
+  }
+  if (dryRun) {
+    lines.push(`(dry-run: no writes performed)`);
+  } else {
+    lines.push(`commits: ${report.commits.length}`);
+    for (const c of report.commits) lines.push(`  ${c}`);
+  }
+  console.log(lines.join('\n'));
+
+  process.stdout.write(`\n${JSON.stringify(reportToJson(report), null, 2)}\n`);
+}
+
+function reportToJson(report: ImportReport): unknown {
+  return {
+    runAt: report.runAt,
+    sourceSha256: report.sourceSha256,
+    entities: report.entities,
+    warnings: report.warnings,
+    commits: report.commits,
+  };
+}
+
+const isMain =
+  process.argv[1] !== undefined &&
+  import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/'));
+
+if (isMain) {
+  main().catch((err: unknown) => {
+    console.error('[import-laddr] failed:', err);
+    process.exit(1);
+  });
+}