feat(copilot): build the docs vfs from a generated manifest, rescope search_docs

.github/workflows/test-build.yml

@@ -150,6 +150,9 @@ jobs:
       - name: Verify agent stream capability docs are in sync
         run: bun run agent-stream-docs:check
 
+      - name: Verify docs manifest is in sync
+        run: bun run docs-manifest:check
+
       - name: Migration safety (zero-downtime) audit
         run: |
           if [ "${{ github.event_name }}" = "pull_request" ]; then

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx

@@ -746,7 +746,7 @@ export function WorkspaceResourceDisplay({
 
     return {
       type: toMothershipResourceType(data.type),
-      id: data.id ?? fileFromPath?.id ?? data.path ?? '',
+      id: [data.id, fileFromPath?.id, data.path].find((value) => value?.trim()) ?? '',
       title,
       ...(data.type === 'file' && data.path ? { path: data.path } : {}),
     }

apps/sim/lib/copilot/chat/post.ts

@@ -1137,6 +1137,7 @@ export async function handleUnifiedChatPost(req: NextRequest) {
     otelRoot?.finish('error', error)
 
     if (isZodError(error)) {
+      logger.warn(`[${requestId}] Chat request failed validation`, { issues: error.issues })
       return validationErrorResponse(error, 'Invalid request data')
     }
 

apps/sim/lib/copilot/chat/process-contents.ts

@@ -30,7 +30,6 @@ import { getSkillById } from '@/lib/workflows/skills/operations'
 import { listFolders } from '@/lib/workflows/utils'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
 import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check'
-import { escapeRegExp } from '@/executor/constants'
 import type { ChatContext } from '@/stores/panel'
 
 type AgentContextType =
@@ -43,7 +42,6 @@ type AgentContextType =
   | 'table'
   | 'file'
   | 'workflow_block'
-  | 'docs'
   | 'folder'
   | 'filefolder'
   | 'active_resource'
@@ -69,7 +67,8 @@ const logger = createLogger('ProcessContents')
 export async function processContextsServer(
   contexts: ChatContext[] | undefined,
   userId: string,
-  userMessage?: string,
+  /** Retained for call-site compatibility; unused while @docs tagging is disabled. */
+  _userMessage: string | undefined,
   currentWorkspaceId?: string,
   chatId?: string
 ): Promise<AgentContext[]> {
@@ -202,21 +201,9 @@ export async function processContextsServer(
           path: result.path,
         }
       }
-      if (ctx.kind === 'docs') {
-        try {
-          const { searchDocumentationServerTool } = await import(
-            '@/lib/copilot/tools/server/docs/search-documentation'
-          )
-          const rawQuery = (userMessage || '').trim() || ctx.label || 'Sim documentation'
-          const query = sanitizeMessageForDocs(rawQuery, contexts)
-          const res = await searchDocumentationServerTool.execute({ query, topK: 10 })
-          const content = JSON.stringify(res?.results || [])
-          return { type: 'docs', tag: ctx.label ? `@${ctx.label}` : '@', content }
-        } catch (e) {
-          logger.error('Failed to process docs context', e)
-          return null
-        }
-      }
+      // `docs` contexts are intentionally inert: @docs tagging is disabled while
+      // the docs corpus moves to the `docs/` VFS tree. A tagged context resolves
+      // to nothing and is filtered out below.
       return null
     } catch (error) {
       logger.error('Failed processing context (server)', { ctx, error })
@@ -238,53 +225,6 @@ export async function processContextsServer(
   return filtered
 }
 
-function sanitizeMessageForDocs(rawMessage: string, contexts: ChatContext[] | undefined): string {
-  if (!rawMessage) return ''
-  if (!Array.isArray(contexts) || contexts.length === 0) {
-    // No context mapping; conservatively strip all @mentions-like tokens
-    const stripped = rawMessage
-      .replace(/(^|\s)@([^\s]+)/g, ' ')
-      .replace(/\s{2,}/g, ' ')
-      .trim()
-    return stripped
-  }
-
-  // Gather labels by kind
-  const blockLabels = new Set(
-    contexts
-      .filter((c) => c.kind === 'blocks')
-      .map((c) => c.label)
-      .filter((l): l is string => typeof l === 'string' && l.length > 0)
-  )
-  const nonBlockLabels = new Set(
-    contexts
-      .filter((c) => c.kind !== 'blocks')
-      .map((c) => c.label)
-      .filter((l): l is string => typeof l === 'string' && l.length > 0)
-  )
-
-  let result = rawMessage
-
-  // 1) Remove all non-block mentions entirely
-  for (const label of nonBlockLabels) {
-    const pattern = new RegExp(`(^|\\s)@${escapeRegExp(label)}(?!\\S)`, 'g')
-    result = result.replace(pattern, ' ')
-  }
-
-  // 2) For block mentions, strip the '@' but keep the block name
-  for (const label of blockLabels) {
-    const pattern = new RegExp(`@${escapeRegExp(label)}(?!\\S)`, 'g')
-    result = result.replace(pattern, label)
-  }
-
-  // 3) Remove any remaining @mentions (unknown or not in contexts)
-  result = result.replace(/(^|\s)@([^\s]+)/g, ' ')
-
-  // Normalize whitespace
-  result = result.replace(/\s{2,}/g, ' ').trim()
-  return result
-}
-
 async function processSkillFromDb(
   skillId: string,
   workspaceId: string,

apps/sim/lib/copilot/docs/docs-corpus.test.ts

@@ -0,0 +1,153 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  couldMatchDocsScope,
+  DocsCorpusError,
+  globDocs,
+  grepDocsPage,
+  isDocsPath,
+  readDocsPage,
+} from '@/lib/copilot/docs/docs-corpus'
+import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
+
+const SAMPLE_PAGE = DOCS_MANIFEST.find((path) => path === 'workflows/blocks/agent.mdx')
+
+describe('docs corpus scoping', () => {
+  it('recognizes docs paths', () => {
+    expect(isDocsPath('docs/workflows.mdx')).toBe(true)
+    expect(isDocsPath('docs')).toBe(true)
+    expect(isDocsPath('/docs/workflows.mdx')).toBe(true)
+    expect(isDocsPath('workflows.mdx')).toBe(false)
+    expect(isDocsPath('files/report.pdf')).toBe(false)
+    expect(isDocsPath('docsomething/x')).toBe(false)
+    expect(isDocsPath(undefined)).toBe(false)
+  })
+
+  it('is opt-in: only an explicit docs/ pattern can match', () => {
+    expect(couldMatchDocsScope('docs/**')).toBe(true)
+    expect(couldMatchDocsScope('docs/workflows/**')).toBe(true)
+    expect(couldMatchDocsScope('**')).toBe(false)
+    expect(couldMatchDocsScope('**/*.mdx')).toBe(false)
+    expect(couldMatchDocsScope('*')).toBe(false)
+    expect(couldMatchDocsScope(undefined)).toBe(false)
+  })
+})
+
+describe('globDocs', () => {
+  it('lists the whole corpus under docs/**', () => {
+    const files = globDocs('docs/**')
+    expect(files.length).toBeGreaterThan(DOCS_MANIFEST.length)
+    expect(files).toContain('docs/workflows/blocks/agent.mdx')
+    expect(files).toContain('docs/workflows/blocks')
+  })
+
+  it('scopes to a section', () => {
+    const files = globDocs('docs/integrations/*.mdx')
+    expect(files).toContain('docs/integrations/gmail.mdx')
+    expect(files.every((path) => path.startsWith('docs/integrations/'))).toBe(true)
+  })
+
+  it('excludes academy and api-reference', () => {
+    expect(globDocs('docs/academy/**')).toEqual([])
+    expect(globDocs('docs/api-reference/**')).toEqual([])
+  })
+
+  it('maps section index pages onto their parent URL path', () => {
+    expect(globDocs('docs/workflows.mdx')).toEqual(['docs/workflows.mdx'])
+    expect(globDocs('docs/workflows/index.mdx')).toEqual([])
+  })
+})
+
+describe('readDocsPage', () => {
+  const fetchMock = vi.fn()
+
+  beforeEach(() => {
+    fetchMock.mockReset()
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('fetches the manifest path verbatim from the docs site', async () => {
+    expect(SAMPLE_PAGE).toBeDefined()
+    fetchMock.mockResolvedValue({ ok: true, status: 200, text: async () => '# Agent\n\nbody' })
+
+    const page = await readDocsPage(`docs/${SAMPLE_PAGE}`)
+
+    expect(fetchMock).toHaveBeenCalledOnce()
+    expect(fetchMock.mock.calls[0][0]).toBe(`https://docs.sim.ai/${SAMPLE_PAGE}`)
+    expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 })
+  })
+
+  it('rejects an unknown page without fetching', async () => {
+    await expect(readDocsPage('docs/not-a-real-page.mdx')).rejects.toThrow(DocsCorpusError)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('points a directory read at glob', async () => {
+    await expect(readDocsPage('docs/workflows/blocks')).rejects.toThrow(/is a directory/)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('surfaces a docs-site outage as a retryable error', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 502, text: async () => '' })
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+  })
+
+  it('treats a network failure as retryable', async () => {
+    fetchMock.mockRejectedValue(new Error('socket hang up'))
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+  })
+
+  it('reports a page the site no longer serves as permanent, not retryable', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 404, text: async () => '' })
+    const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e)
+    expect(error).toBeInstanceOf(DocsCorpusError)
+    expect(error.message).toMatch(/does not serve it/)
+    expect(error.message).toMatch(/retrying will not help/)
+    expect(error.message).not.toMatch(/temporarily unavailable/)
+  })
+
+  it('still treats 429 as retryable rather than permanent', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 429, text: async () => '' })
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+  })
+})
+
+describe('grepDocsPage', () => {
+  const fetchMock = vi.fn()
+
+  beforeEach(() => {
+    fetchMock.mockReset()
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('greps exactly one page', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'intro line\nsystemPrompt matters\ntail',
+    })
+
+    const matches = await grepDocsPage(`docs/${SAMPLE_PAGE}`, 'systemPrompt')
+
+    expect(fetchMock).toHaveBeenCalledOnce()
+    expect(matches).toEqual([
+      { path: `docs/${SAMPLE_PAGE}`, line: 2, content: 'systemPrompt matters' },
+    ])
+  })
+
+  it('refuses a multi-page scope so one grep is never hundreds of fetches', async () => {
+    await expect(grepDocsPage('docs/', 'cron')).rejects.toThrow(/single page/)
+    await expect(grepDocsPage('docs/workflows', 'cron')).rejects.toThrow(/single page/)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+})