Merge remote-tracking branch 'origin/staging' into improvement/table-rename-column

# Conflicts:
#	packages/db/migrations/meta/0227_snapshot.json
#	packages/db/migrations/meta/_journal.json

Author: TheodoreSpeaks

.agents/skills/add-block/SKILL.md

@@ -627,19 +627,78 @@ export const ServiceV2Block: BlockConfig = {
 }
 ```
 
+## Block Metadata (BlockMeta)
+
+Every integration block **must** export a `{Service}BlockMeta` object at the bottom of the block file. This metadata drives the integration catalog, tag filters, and workflow template suggestions shown to users.
+
+### Structure
+
+```typescript
+import type { BlockConfig, BlockMeta } from '@/blocks/types'
+
+// ... block definition above ...
+
+export const {Service}BlockMeta = {
+  tags: ['messaging', 'automation'],   // Same tags as the block's tags field
+  templates: [                          // Optional but strongly encouraged
+    {
+      icon: {Service}Icon,
+      title: '{Service} use-case title',
+      prompt: 'Build a workflow that ...',
+      modules: ['agent', 'workflows'],  // Modules the template uses
+      category: 'productivity',         // Template category
+      tags: ['automation'],             // Template-level tags
+      alsoIntegrations: ['slack'],      // Other blocks referenced in the prompt (optional)
+    },
+  ],
+} as const satisfies BlockMeta
+```
+
+### Rules
+
+- **Import `BlockMeta`** from `@/blocks/types` alongside `BlockConfig`
+- **`tags`** must match the `tags` array on the block config exactly
+- **Templates are optional** but should be added for any integration that has a recognizable use case — aim for 2–4 templates per block
+- **Template `prompt`** should start with "Build a workflow that..." or "Create a workflow that..." and be concrete enough to generate a real workflow in Mothership
+- **Template `modules`** lists the Sim modules the template relies on: `'knowledge-base' | 'tables' | 'files' | 'workflows' | 'scheduled' | 'agent'`
+- **Template `category`** is one of: `'popular' | 'sales' | 'support' | 'engineering' | 'marketing' | 'productivity' | 'operations'`
+- **`alsoIntegrations`** names other block types (e.g. `'slack'`, `'linear'`) referenced in the template prompt — helps the catalog surface this template when those blocks are selected
+- Place the export **after** the main `{Service}Block` export, at the very bottom of the file
+
+### Register in the blocksMeta object
+
+After adding `{Service}BlockMeta` to the block file, register it in `apps/sim/blocks/registry.ts`:
+
+```typescript
+// Add import (alongside the block import, alphabetically)
+import { ServiceBlock, ServiceBlockMeta } from '@/blocks/blocks/service'
+
+// Add to blocksMeta object (alphabetically)
+export const blocksMeta = {
+  // ... existing entries ...
+  service: ServiceBlockMeta,
+}
+```
+
 ## Registering Blocks
 
 After creating the block, remind the user to:
-1. Import in `apps/sim/blocks/registry.ts`
+1. Import `{Service}Block` and `{Service}BlockMeta` in `apps/sim/blocks/registry.ts`
 2. Add to the `registry` object (alphabetically):
+3. Add to the `blocksMeta` object (alphabetically):
 
 ```typescript
-import { ServiceBlock } from '@/blocks/blocks/service'
+import { ServiceBlock, ServiceBlockMeta } from '@/blocks/blocks/service'
 
 export const registry: Record<string, BlockConfig> = {
   // ... existing blocks ...
   service: ServiceBlock,
 }
+
+export const blocksMeta = {
+  // ... existing entries ...
+  service: ServiceBlockMeta,
+}
 ```
 
 ## Complete Example
@@ -827,7 +886,10 @@ All tool IDs referenced in `tools.access` and returned by `tools.config.tool` MU
 - [ ] Tools.access lists all tool IDs (snake_case)
 - [ ] Tools.config.tool returns correct tool ID (snake_case)
 - [ ] Outputs match tool outputs
-- [ ] Block registered in registry.ts
+- [ ] Block registered in `registry.ts` blocks object (alphabetically)
+- [ ] `{Service}BlockMeta` exported at bottom of block file with `tags` and `templates`
+- [ ] `BlockMeta` imported from `@/blocks/types` alongside `BlockConfig`
+- [ ] Block meta registered in `registry.ts` blocksMeta object (alphabetically)
 - [ ] If icon missing: asked user to provide SVG
 - [ ] If triggers exist: `triggers` config set, trigger subBlocks spread
 - [ ] Optional/rarely-used fields set to `mode: 'advanced'`

.agents/skills/add-integration/SKILL.md

@@ -128,7 +128,7 @@ export const {service}{Action}Tool: ToolConfig<Params, Response> = {
 ### Block Structure
 ```typescript
 import { {Service}Icon } from '@/components/icons'
-import type { BlockConfig } from '@/blocks/types'
+import type { BlockConfig, BlockMeta } from '@/blocks/types'
 import { AuthMode } from '@/blocks/types'
 import { getScopesForService } from '@/lib/oauth/utils'
 
@@ -177,8 +177,32 @@ export const {Service}Block: BlockConfig = {
 
   outputs: { /* ... */ },
 }
+
+export const {Service}BlockMeta = {
+  tags: ['tag1', 'tag2'],  // IntegrationTag values matching the service's capabilities
+  templates: [
+    {
+      icon: {Service}Icon,
+      title: '{Service} use-case title',
+      prompt: 'Build a workflow that ...',
+      modules: ['agent', 'workflows'],
+      category: 'productivity',
+      tags: ['automation'],
+      alsoIntegrations: ['slack'],  // Optional: other blocks referenced in the prompt
+    },
+  ],
+} as const satisfies BlockMeta
 ```
 
+### BlockMeta rules
+
+- **Tags**: Use `IntegrationTag` values from `@/blocks/types`. Do NOT add a `tags` field to the `BlockConfig` object — tags belong only in `BlockMeta`.
+- **`integrationType`**: Must be a valid `IntegrationType` enum value (AI, Analytics, Commerce, Communication, Databases, DevOps, Documents, Email, HR, Marketing, Observability, Productivity, Sales, Search, Security, Support). Never invent a value that doesn't exist in the enum.
+- **Templates**: Aim for 2–4 templates per integration. Prompts should be concrete enough to generate a real workflow in Mothership. Start with "Build a workflow that..." or "Create a workflow that...".
+- **`alsoIntegrations`**: List other block type IDs referenced in the template prompt (e.g. `'slack'`, `'linear'`).
+- Place `{Service}BlockMeta` at the very bottom of the file, after the main block export.
+- Register it in both the import and the `blocksMeta` object in `registry.ts`.
+
 ### Key SubBlock Patterns
 
 **Condition-based visibility:**
@@ -359,14 +383,20 @@ export const tools: Record<string, ToolConfig> = {
 ### Block Registry (`apps/sim/blocks/registry.ts`)
 
 ```typescript
-// Add import (alphabetically)
-import { {Service}Block } from '@/blocks/blocks/{service}'
+// Add import (alphabetically) — include BlockMeta
+import { {Service}Block, {Service}BlockMeta } from '@/blocks/blocks/{service}'
 
-// Add to registry (alphabetically)
+// Add to blocks registry (alphabetically)
 export const registry: Record<string, BlockConfig> = {
   // ... existing blocks ...
   {service}: {Service}Block,
 }
+
+// Add to blocksMeta registry (alphabetically)
+export const blocksMeta = {
+  // ... existing entries ...
+  {service}: {Service}BlockMeta,
+}
 ```
 
 ### Trigger Registry (`apps/sim/triggers/registry.ts`) - If triggers exist
@@ -433,7 +463,12 @@ If creating V2 versions (API-aligned outputs):
 - [ ] Configured tools.access with all tool IDs
 - [ ] Configured tools.config.tool selector
 - [ ] Defined outputs matching tool outputs
-- [ ] Registered block in `blocks/registry.ts`
+- [ ] `integrationType` uses a valid `IntegrationType` enum value (no invented values)
+- [ ] No `tags` field on the `BlockConfig` object (tags live only in `BlockMeta`)
+- [ ] `{Service}BlockMeta` exported at bottom of file with `tags` and `templates`
+- [ ] `BlockMeta` type imported from `@/blocks/types` alongside `BlockConfig`
+- [ ] Block registered in `blocks/registry.ts` blocks object (alphabetically)
+- [ ] Block meta registered in `blocks/registry.ts` blocksMeta object (alphabetically)
 - [ ] If triggers: set `triggers.enabled` and `triggers.available`
 - [ ] If triggers: spread trigger subBlocks with `getTrigger()`
 

.claude/commands/add-block.md

@@ -429,7 +429,7 @@ Maps multiple UI fields to a single serialized parameter:
 
 **Critical constraints:**
 - `canonicalParamId` must NOT match any other subblock's `id` in the same block (causes conflicts)
-- `canonicalParamId` must be unique per block (only one basic/advanced pair per canonicalParamId)
+- A `canonicalParamId` links exactly one basic/advanced pair for a single logical parameter. Do NOT reuse the same `canonicalParamId` for different parameters, even under mutually-exclusive conditions/operations
 - ONLY use `canonicalParamId` to link basic/advanced mode alternatives for the same logical parameter
 - Do NOT use it for any other purpose
 
@@ -519,7 +519,10 @@ tools: {
 Block outputs only support:
 - `type` - The data type ('string', 'number', 'boolean', 'json', 'array')
 - `description` - Human readable description
-- Nested object structure (for complex types)
+- `condition` - Optional visibility condition
+- `hiddenFromDisplay` - Optional flag to hide from the output display
+
+**Nested object/`properties` outputs are tool-output-only and will fail TypeScript at build time on block outputs.** For complex shapes use `type: 'json'` and describe the inner fields in the `description` string.
 
 ```typescript
 outputs: {
@@ -542,7 +545,7 @@ outputs: {
 
 ### Typed JSON Outputs
 
-When using `type: 'json'` and you know the object shape in advance, **describe the inner fields in the description** so downstream blocks know what properties are available. For well-known, stable objects, use nested output definitions instead:
+When using `type: 'json'` and you know the object shape in advance, **describe the inner fields in the description** so downstream blocks know what properties are available. Block outputs have no nested `properties` form — always keep the output flat and put the shape in the `description`:
 
 ```typescript
 outputs: {
@@ -554,26 +557,10 @@ outputs: {
     type: 'json',
     description: 'Zone plan information (id, name, price, currency, frequency, is_subscribed)',
   },
-
-  // BEST: Use nested output definition when the shape is stable and well-known
-  plan: {
-    id: { type: 'string', description: 'Plan identifier' },
-    name: { type: 'string', description: 'Plan name' },
-    price: { type: 'number', description: 'Plan price' },
-    currency: { type: 'string', description: 'Price currency' },
-  },
 }
 ```
 
-Use the nested pattern when:
-- The object has a small, stable set of fields (< 10)
-- Downstream blocks will commonly access specific properties
-- The API response shape is well-documented and unlikely to change
-
-Use `type: 'json'` with a descriptive string when:
-- The object has many fields or a dynamic shape
-- It represents a list/array of items
-- The shape varies by operation
+Nested object outputs (`plan: { id: { type: 'string' }, ... }`) are a **tool-output** feature only — `OutputFieldDefinition` for blocks does not allow them and they fail TypeScript at build time.
 
 ## V2 Block Pattern
 
@@ -798,6 +785,33 @@ Use `wandConfig` for fields that are hard to fill out manually, such as timestam
 
 All tool IDs referenced in `tools.access` and returned by `tools.config.tool` MUST use `snake_case` (e.g., `x_create_tweet`, `slack_send_message`). Never use camelCase or PascalCase.
 
+## BlockMeta (Required)
+
+Every block file must export a `{Service}BlockMeta` alongside the block — **minimum 7 templates**. Look at existing examples in `apps/sim/blocks/blocks/` (e.g. `browser_use.ts`, `google_sheets.ts`) for the pattern.
+
+```typescript
+import type { BlockMeta } from '@/blocks/types'
+
+export const {Service}BlockMeta = {
+  tags: ['tag1', 'tag2'],                  // IntegrationTag[]
+  templates: [
+    {
+      icon: {Service}Icon,
+      title: '{Service} <use-case>',        // 2–5 words
+      prompt: 'Build a workflow that...',   // specific use case, 1–3 sentences
+      modules: ['agent', 'workflows'],      // 'agent' | 'workflows' | 'tables' | 'files' | 'scheduled' | 'knowledge-base'
+      category: 'operations',              // 'operations' | 'marketing' | 'sales' | 'engineering' | 'productivity' | 'support' | 'popular'
+      tags: ['automation'],
+      alsoIntegrations: ['slack'],         // optional — other block IDs referenced in the prompt
+      featured: true,                      // optional
+    },
+    // ... at least 6 more
+  ],
+} as const satisfies BlockMeta
+```
+
+Derive templates from the service's real use cases. Each prompt should name a concrete trigger, transformation, and output — not a generic description of what the service does.
+
 ## Checklist Before Finishing
 
 - [ ] `integrationType` is set to the correct `IntegrationType` enum value
@@ -816,6 +830,7 @@ All tool IDs referenced in `tools.access` and returned by `tools.config.tool` MU
 - [ ] If triggers exist: `triggers` config set, trigger subBlocks spread
 - [ ] Optional/rarely-used fields set to `mode: 'advanced'`
 - [ ] Timestamps and complex inputs have `wandConfig` enabled
+- [ ] Exported `{Service}BlockMeta` with at least 7 templates
 
 ## Final Validation (Required)
 
@@ -829,3 +844,4 @@ After creating the block, you MUST validate it against every tool it references:
    - Type coercions in `tools.config.params` for any params that need conversion (Number(), Boolean(), JSON.parse())
 3. **Verify block outputs** cover the key fields returned by all tools
 4. **Verify conditions** — each subBlock should only show for the operations that actually use it
+5. **Verify `{Service}BlockMeta` is exported** with at least 7 templates, each having `icon`, `title`, `prompt`, `modules`, `category`, and `tags`

.claude/commands/add-connector.md

@@ -67,7 +67,7 @@ export const {service}Connector: ConnectorConfig = {
 
   configFields: [
     // Rendered dynamically by the add-connector modal UI
-    // Supports 'short-input' and 'dropdown' types
+    // Supports 'short-input', 'dropdown', and 'selector' types — see ConfigField Types below
   ],
 
   listDocuments: async (accessToken, sourceConfig, cursor) => {
@@ -408,6 +408,16 @@ Each entry has:
 Users can opt out of specific tags in the modal. Disabled IDs are stored in `sourceConfig.disabledTagIds`.
 The assigned mapping (`semantic id → slot`) is stored in `sourceConfig.tagSlotMapping`.
 
+## `@/connectors/utils` Helpers
+
+Reuse these instead of inlining the same logic (the validator enforces them):
+
+- `htmlToPlainText(html)` — strip HTML to plain text before indexing `ExternalDocument.content`. Never index raw HTML.
+- `computeContentHash(content)` — stable content hash for change detection.
+- `parseTagDate(value)` — parse to a valid `Date` or `undefined` (guards Invalid Date). Use in `mapTags` for date fields.
+- `joinTagArray(value)` — validate an array and join to a comma-separated string, or `undefined`. Use in `mapTags` for array/label fields.
+- `parseMultiValue(value)` — normalize a value into a `string[]`.
+
 ## mapTags — Metadata to Semantic Keys
 
 Maps source metadata to semantic tag keys. Required if `tagDefinitions` is set.
@@ -416,25 +426,27 @@ using the `tagSlotMapping` stored on the connector.
 
 Return keys must match the `id` values declared in `tagDefinitions`.
 
+Use the `@/connectors/utils` helpers for the common transforms — don't hand-roll date/array validation:
+
 ```typescript
+import { joinTagArray, parseTagDate } from '@/connectors/utils'
+
 mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
   const result: Record<string, unknown> = {}
 
-  // Validate arrays before casting — metadata may be malformed
-  const labels = Array.isArray(metadata.labels) ? (metadata.labels as string[]) : []
-  if (labels.length > 0) result.labels = labels.join(', ')
+  // joinTagArray validates the array and joins to a comma-separated string (undefined if empty)
+  const labels = joinTagArray(metadata.labels)
+  if (labels) result.labels = labels
 
   // Validate numbers — guard against NaN
   if (metadata.version != null) {
     const num = Number(metadata.version)
     if (!Number.isNaN(num)) result.version = num
   }
 
-  // Validate dates — guard against Invalid Date
-  if (typeof metadata.lastModified === 'string') {
-    const date = new Date(metadata.lastModified)
-    if (!Number.isNaN(date.getTime())) result.lastModified = date
-  }
+  // parseTagDate returns a valid Date or undefined (guards against Invalid Date)
+  const lastModified = parseTagDate(metadata.lastModified)
+  if (lastModified) result.lastModified = lastModified
 
   return result
 }

.claude/commands/add-hosted-key.md

@@ -11,7 +11,7 @@ When a tool has hosted key support, Sim provides its own API key if the user has
 
 | Step | What | Where |
 |------|------|-------|
-| 1 | Register BYOK provider ID | `tools/types.ts`, `app/api/workspaces/[id]/byok-keys/route.ts` |
+| 1 | Register BYOK provider ID | `tools/types.ts`, `lib/api/contracts/byok-keys.ts` |
 | 2 | Research the API's pricing and rate limits | API docs / pricing page (before writing any code) |
 | 3 | Add `hosting` config to the tool | `tools/{service}/{action}.ts` |
 | 4 | Hide API key field when hosted | `blocks/blocks/{service}.ts` |
@@ -30,10 +30,15 @@ export type BYOKProviderId =
   | 'your_service'
 ```
 
-Then add it to `VALID_PROVIDERS` in `app/api/workspaces/[id]/byok-keys/route.ts`:
+Then add the same provider id to the `byokProviderIdSchema` enum in `lib/api/contracts/byok-keys.ts` (this is what the byok-keys route validates against):
 
 ```typescript
-const VALID_PROVIDERS = ['openai', 'anthropic', 'google', 'mistral', 'your_service'] as const
+export const byokProviderIdSchema = z.enum([
+  'openai',
+  'anthropic',
+  // ...existing providers
+  'your_service',
+])
 ```
 
 ## Step 2: Research the API's Pricing Model and Rate Limits
@@ -230,7 +235,7 @@ Both subblocks share the same `id: 'apiKey'`, so the same value flows to the too
 To exclude multiple operations, use an array: `{ field: 'operation', value: ['op_a', 'op_b'] }`.
 
 **Reference implementations:**
-- **Exa** (`blocks/blocks/exa.ts`): `research` operation excluded from hosting — lines 309-329
+- **Exa** (`blocks/blocks/exa.ts`): `exa_research` operation excluded from hosting — duplicate `apiKey` pair around lines ~348-365
 - **Google Maps** (`blocks/blocks/google_maps.ts`): `speed_limits` operation excluded from hosting (deprecated Roads API)
 
 ## Step 5: Add to the BYOK Settings UI
@@ -284,7 +289,7 @@ This summary helps reviewers verify that the pricing and rate limiting are well-
 ## Checklist
 
 - [ ] Provider added to `BYOKProviderId` in `tools/types.ts`
-- [ ] Provider added to `VALID_PROVIDERS` in the BYOK keys API route
+- [ ] Provider added to `byokProviderIdSchema` enum in `lib/api/contracts/byok-keys.ts`
 - [ ] API pricing docs researched — understand per-unit cost and whether the API reports cost in responses
 - [ ] API rate limits researched — understand RPM/TPM limits, per-key vs per-account, and plan tiers
 - [ ] `hosting` config added to the tool with `envKeyPrefix`, `apiKeyParam`, `byokProviderId`, `pricing`, and `rateLimit`