diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index c50e076a317..8d24d4c27fd 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -159,6 +159,9 @@ jobs: - name: Verify agent stream capability docs are in sync run: bun run agent-stream-docs:check + - name: Verify docs manifest is in sync + run: bun run docs-manifest:check + - name: Migration safety (zero-downtime) audit run: | if [ "${{ github.event_name }}" = "pull_request" ]; then diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 2dd4e87562d..117d7203175 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -1353,7 +1353,7 @@ export function WorkspaceResourceDisplay({ return { type: toMothershipResourceType(data.type), - id: data.id ?? fileFromPath?.id ?? data.path ?? '', + id: [data.id, fileFromPath?.id, data.path].find((value) => value?.trim()) ?? '', title, ...(data.type === 'file' && data.path ? { path: data.path } : {}), } diff --git a/apps/sim/lib/copilot/chat/post.ts b/apps/sim/lib/copilot/chat/post.ts index 96878919b87..4082e0e2f37 100644 --- a/apps/sim/lib/copilot/chat/post.ts +++ b/apps/sim/lib/copilot/chat/post.ts @@ -143,6 +143,8 @@ const ChatContextSchema = z.object({ 'logs', 'workflow_block', 'knowledge', + // Inert today: processContextsServer drops docs contexts. Kept in the + // schema so a future @Docs mention can resolve through search_docs. 'docs', 'table', 'file', @@ -1296,6 +1298,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { otelRoot?.finish('error', error) if (isZodError(error)) { + logger.warn(`[${requestId}] Chat request failed validation`, { issues: error.issues }) return validationErrorResponse(error, 'Invalid request data') } diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index bb2172e71f3..fc1a4ee3656 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -78,6 +78,23 @@ describe('processContextsServer - skill contexts', () => { }) }) +describe('processContextsServer - docs contexts', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('resolves a tagged docs context to nothing while @docs tagging is disabled', async () => { + const result = await processContextsServer( + [{ kind: 'docs', label: 'Docs' } as ChatContext], + 'user-1', + 'how do loops work @Docs', + 'ws-1' + ) + + expect(result).toEqual([]) + }) +}) + describe('processContextsServer - MCP contexts', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index 54fce5d321c..a3b03e0354f 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -30,7 +30,6 @@ import { getSkillById } from '@/lib/workflows/skills/operations' import { listFolders } from '@/lib/workflows/utils' import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' -import { escapeRegExp } from '@/executor/constants' import type { ChatContext } from '@/stores/panel' type AgentContextType = @@ -43,7 +42,6 @@ type AgentContextType = | 'table' | 'file' | 'workflow_block' - | 'docs' | 'folder' | 'filefolder' | 'active_resource' @@ -69,7 +67,8 @@ const logger = createLogger('ProcessContents') export async function processContextsServer( contexts: ChatContext[] | undefined, userId: string, - userMessage?: string, + /** Retained for call-site compatibility; unused while @docs tagging is disabled. */ + _userMessage: string | undefined, currentWorkspaceId?: string, chatId?: string ): Promise { @@ -220,21 +219,11 @@ export async function processContextsServer( path: result.path, } } - if (ctx.kind === 'docs') { - try { - const { searchDocumentationServerTool } = await import( - '@/lib/copilot/tools/server/docs/search-documentation' - ) - const rawQuery = (userMessage || '').trim() || ctx.label || 'Sim documentation' - const query = sanitizeMessageForDocs(rawQuery, contexts) - const res = await searchDocumentationServerTool.execute({ query, topK: 10 }) - const content = JSON.stringify(res?.results || []) - return { type: 'docs', tag: ctx.label ? `@${ctx.label}` : '@', content } - } catch (e) { - logger.error('Failed to process docs context', e) - return null - } - } + // `docs` contexts are intentionally inert: @docs tagging is disabled now + // that the docs corpus lives in the `docs/` VFS tree. If @Docs returns, + // resolve it here through searchDocs (the search_docs backend) rather + // than reviving the retired whole-message pre-search. A tagged context + // resolves to nothing and is filtered out below. return null } catch (error) { logger.error('Failed processing context (server)', { ctx, error }) @@ -256,53 +245,6 @@ export async function processContextsServer( return filtered } -function sanitizeMessageForDocs(rawMessage: string, contexts: ChatContext[] | undefined): string { - if (!rawMessage) return '' - if (!Array.isArray(contexts) || contexts.length === 0) { - // No context mapping; conservatively strip all @mentions-like tokens - const stripped = rawMessage - .replace(/(^|\s)@([^\s]+)/g, ' ') - .replace(/\s{2,}/g, ' ') - .trim() - return stripped - } - - // Gather labels by kind - const blockLabels = new Set( - contexts - .filter((c) => c.kind === 'blocks') - .map((c) => c.label) - .filter((l): l is string => typeof l === 'string' && l.length > 0) - ) - const nonBlockLabels = new Set( - contexts - .filter((c) => c.kind !== 'blocks') - .map((c) => c.label) - .filter((l): l is string => typeof l === 'string' && l.length > 0) - ) - - let result = rawMessage - - // 1) Remove all non-block mentions entirely - for (const label of nonBlockLabels) { - const pattern = new RegExp(`(^|\\s)@${escapeRegExp(label)}(?!\\S)`, 'g') - result = result.replace(pattern, ' ') - } - - // 2) For block mentions, strip the '@' but keep the block name - for (const label of blockLabels) { - const pattern = new RegExp(`@${escapeRegExp(label)}(?!\\S)`, 'g') - result = result.replace(pattern, label) - } - - // 3) Remove any remaining @mentions (unknown or not in contexts) - result = result.replace(/(^|\s)@([^\s]+)/g, ' ') - - // Normalize whitespace - result = result.replace(/\s{2,}/g, ' ').trim() - return result -} - async function processSkillFromDb( skillId: string, workspaceId: string, diff --git a/apps/sim/lib/copilot/docs/docs-corpus.test.ts b/apps/sim/lib/copilot/docs/docs-corpus.test.ts new file mode 100644 index 00000000000..f3dd5a8f4ad --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-corpus.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + couldMatchDocsScope, + DocsCorpusError, + globDocs, + grepDocsPage, + isDocsPath, + readDocsPage, +} from '@/lib/copilot/docs/docs-corpus' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' + +const SAMPLE_PAGE = DOCS_MANIFEST.find((path) => path === 'workflows/blocks/agent.mdx') + +describe('docs corpus scoping', () => { + it('recognizes docs paths', () => { + expect(isDocsPath('docs/workflows.mdx')).toBe(true) + expect(isDocsPath('docs')).toBe(true) + expect(isDocsPath('/docs/workflows.mdx')).toBe(true) + expect(isDocsPath('workflows.mdx')).toBe(false) + expect(isDocsPath('files/report.pdf')).toBe(false) + expect(isDocsPath('docsomething/x')).toBe(false) + expect(isDocsPath(undefined)).toBe(false) + }) + + it('is opt-in: only an explicit docs/ pattern can match', () => { + expect(couldMatchDocsScope('docs/**')).toBe(true) + expect(couldMatchDocsScope('docs/workflows/**')).toBe(true) + expect(couldMatchDocsScope('**')).toBe(false) + expect(couldMatchDocsScope('**/*.mdx')).toBe(false) + expect(couldMatchDocsScope('*')).toBe(false) + expect(couldMatchDocsScope(undefined)).toBe(false) + }) +}) + +describe('globDocs', () => { + it('lists the whole corpus under docs/**', () => { + const files = globDocs('docs/**') + expect(files.length).toBeGreaterThan(DOCS_MANIFEST.length) + expect(files).toContain('docs/workflows/blocks/agent.mdx') + expect(files).toContain('docs/workflows/blocks') + }) + + it('scopes to a section', () => { + const files = globDocs('docs/integrations/*.mdx') + expect(files).toContain('docs/integrations/gmail.mdx') + expect(files.every((path) => path.startsWith('docs/integrations/'))).toBe(true) + }) + + it('excludes academy and api-reference', () => { + expect(globDocs('docs/academy/**')).toEqual([]) + expect(globDocs('docs/api-reference/**')).toEqual([]) + }) + + it('maps section index pages onto their parent URL path', () => { + expect(globDocs('docs/workflows.mdx')).toEqual(['docs/workflows.mdx']) + expect(globDocs('docs/workflows/index.mdx')).toEqual([]) + }) + + it('treats a trailing-slash pattern like the bare directory instead of matching nothing', () => { + expect(globDocs('docs/')).toEqual(['docs']) + expect(globDocs('docs/integrations/')).toEqual(['docs/integrations']) + }) +}) + +describe('readDocsPage', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('fetches the manifest path verbatim from the docs site', async () => { + expect(SAMPLE_PAGE).toBeDefined() + fetchMock.mockResolvedValue({ ok: true, status: 200, text: async () => '# Agent\n\nbody' }) + + const page = await readDocsPage(`docs/${SAMPLE_PAGE}`) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock.mock.calls[0][0]).toBe(`https://docs.sim.ai/${SAMPLE_PAGE}`) + expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 }) + }) + + it('rejects an unknown page without fetching', async () => { + await expect(readDocsPage('docs/not-a-real-page.mdx')).rejects.toThrow(DocsCorpusError) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('points a directory read at glob', async () => { + await expect(readDocsPage('docs/workflows/blocks')).rejects.toThrow(/is a directory/) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('surfaces a docs-site outage as a retryable error', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 502, text: async () => '' }) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/) + }) + + it('treats a network failure as retryable', async () => { + fetchMock.mockRejectedValue(new Error('socket hang up')) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/) + }) + + it('reports a page the site no longer serves as permanent, not retryable', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 404, text: async () => '' }) + const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e) + expect(error).toBeInstanceOf(DocsCorpusError) + expect(error.message).toMatch(/does not serve it/) + expect(error.message).toMatch(/retrying will not help/) + expect(error.message).not.toMatch(/temporarily unavailable/) + }) + + it('still treats 429 as retryable rather than permanent', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 429, text: async () => '' }) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/) + }) +}) + +describe('grepDocsPage', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('greps exactly one page', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => 'intro line\nsystemPrompt matters\ntail', + }) + + const matches = await grepDocsPage(`docs/${SAMPLE_PAGE}`, 'systemPrompt') + + expect(fetchMock).toHaveBeenCalledOnce() + expect(matches).toEqual([ + { path: `docs/${SAMPLE_PAGE}`, line: 2, content: 'systemPrompt matters' }, + ]) + }) + + it('refuses a multi-page scope so one grep is never hundreds of fetches', async () => { + await expect(grepDocsPage('docs/', 'cron')).rejects.toThrow(/single page/) + await expect(grepDocsPage('docs/workflows', 'cron')).rejects.toThrow(/single page/) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-corpus.ts b/apps/sim/lib/copilot/docs/docs-corpus.ts new file mode 100644 index 00000000000..5b8bcbc3213 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-corpus.ts @@ -0,0 +1,190 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { foldDocsIndexPath } from '@/lib/copilot/docs/docs-path' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' +import type { GrepCountEntry, GrepMatch, GrepOptions } from '@/lib/copilot/vfs/operations' +import { glob as globPaths, grepReadResult } from '@/lib/copilot/vfs/operations' + +const logger = createLogger('DocsCorpus') + +/** The public docs site the `docs/` tree is a lazy view of. */ +const DOCS_BASE_URL = 'https://docs.sim.ai' + +/** VFS prefix the docs corpus is mounted at. */ +const DOCS_PREFIX = 'docs/' + +const FETCH_TIMEOUT_MS = 10_000 + +/** + * Thrown for expected, user-facing docs-corpus conditions (unknown page, + * directory path, site unreachable). The VFS handlers return the message as the + * tool error instead of logging an internal failure. + */ +export class DocsCorpusError extends Error { + readonly code = 'DOCS_CORPUS' as const + constructor(message: string) { + super(message) + this.name = 'DocsCorpusError' + } +} + +/** + * Keys-only view of the corpus for glob: every manifest path under `docs/`, + * mapped to empty content. `ops.glob` matches keys and derives the virtual + * directories from them, so this never touches the network. + */ +const docsKeyView: Map = new Map( + DOCS_MANIFEST.map((path) => [`${DOCS_PREFIX}${path}`, '']) +) + +function normalize(path: string): string { + // Trailing slashes are stripped so `docs/` addresses the corpus the same way + // `docs` does — otherwise a trailing-slash glob pattern matches no key and + // silently returns an empty result instead of the corpus listing. + return path.trim().replace(/^\/+/, '').replace(/\/+$/, '') +} + +/** + * True when a read/grep `path` addresses the docs corpus. Deliberately not a + * `path is string` type predicate: the callers chain it ahead of the other + * namespace checks, and a predicate would narrow `path` to `never` in every + * later branch. + */ +export function isDocsPath(path: string | undefined): boolean { + if (!path) return false + const normalized = normalize(path) + return normalized === 'docs' || normalized.startsWith(DOCS_PREFIX) +} + +/** + * True when a glob `pattern` could match the docs corpus. Like `uploads/` and + * `recently-deleted/`, the corpus is opt-in: only a pattern that explicitly + * starts with `docs/` (or is exactly `docs`) sees it, so a broad `**` glob never + * drags 300+ doc pages into the result. Same rule as {@link isDocsPath}; the + * separate name reads correctly at the glob call site. + */ +export function couldMatchDocsScope(pattern: string | undefined): boolean { + return isDocsPath(pattern) +} + +/** Manifest paths (and their virtual directories) matching an explicit `docs/` pattern. */ +export function globDocs(pattern: string): string[] { + return globPaths(docsKeyView, normalize(pattern)) +} + +/** True when `path` is a page in the docs tree. */ +export function isDocsPage(path: string): boolean { + return docsKeyView.has(normalize(path)) +} + +/** + * Map a `docs_embeddings.source_document` (the en-relative mdx file path) back to + * its `docs/` VFS path, applying the same index-page fold as the manifest + * generator. Returns null when the source has no live VFS path — an unmounted + * section (academy, api-reference) or a page deleted since the index was built. + */ +export function docsPathForSourceDocument(sourceDocument: string | null): string | null { + if (!sourceDocument) return null + const path = `${DOCS_PREFIX}${foldDocsIndexPath(sourceDocument.replace(/^\/+/, ''))}` + return docsKeyView.has(path) ? path : null +} + +/** True when `path` is a directory in the docs tree rather than a page. */ +export function isDocsDir(path: string): boolean { + const dir = `${normalize(path).replace(/\/+$/, '')}/` + if (dir === DOCS_PREFIX) return true + for (const key of docsKeyView.keys()) { + if (key.startsWith(dir)) return true + } + return false +} + +export interface DocsPage { + content: string + totalLines: number +} + +/** + * Fetch one docs page's raw markdown from the live site. The manifest path IS + * the URL path (`docs/workflows/blocks/agent.mdx` → + * `https://docs.sim.ai/workflows/blocks/agent.mdx`, which the docs app rewrites + * to its raw-markdown route), so no mapping table is needed. Returns null when + * the page is not in the manifest or the site does not serve it. + */ +type DocsFetchResult = + | { outcome: 'ok'; content: string } + /** The site will not serve this path however many times we ask. */ + | { outcome: 'missing' } + /** Transient: 5xx, 429, network error, or timeout. */ + | { outcome: 'unavailable' } + +async function fetchDocsPage(path: string): Promise { + const key = normalize(path) + if (!docsKeyView.has(key)) return { outcome: 'missing' } + const url = `${DOCS_BASE_URL}/${key.slice(DOCS_PREFIX.length)}` + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + headers: { Accept: 'text/markdown, text/plain' }, + }) + if (!response.ok) { + logger.warn('Docs page fetch returned a non-OK status', { url, status: response.status }) + const permanent = response.status >= 400 && response.status < 500 && response.status !== 429 + return { outcome: permanent ? 'missing' : 'unavailable' } + } + return { outcome: 'ok', content: await response.text() } + } catch (err) { + logger.warn('Docs page fetch failed', { url, error: toError(err).message }) + return { outcome: 'unavailable' } + } +} + +/** + * Read one docs page. Throws {@link DocsCorpusError} for the expected user-facing + * conditions (directory path, unknown page, site unreachable) so the handler can + * surface the message verbatim. + */ +export async function readDocsPage(path: string): Promise { + const key = normalize(path) + if (!docsKeyView.has(key)) { + if (isDocsDir(key)) { + const dir = key.replace(/\/+$/, '') + throw new DocsCorpusError(`${dir} is a directory — glob "${dir}/**" to list its pages.`) + } + throw new DocsCorpusError( + `Docs page not found: ${path}. Use glob("docs/**") to list the docs corpus.` + ) + } + const result = await fetchDocsPage(key) + if (result.outcome === 'missing') { + throw new DocsCorpusError( + `${key} is in the docs index but ${DOCS_BASE_URL} does not serve it — the page was likely moved or removed. Use glob("docs/**") to find the current path; retrying will not help.` + ) + } + if (result.outcome === 'unavailable') { + throw new DocsCorpusError( + `Could not load ${key} from ${DOCS_BASE_URL} — the docs site is temporarily unavailable. Retry shortly.` + ) + } + return { content: result.content, totalLines: result.content.split('\n').length } +} + +/** + * Grep ONE docs page, mirroring how grep over `files/` works: each page is a + * separate fetch from the docs site, so a multi-page grep would mean hundreds of + * requests. A path that is not a single page throws. + */ +export async function grepDocsPage( + path: string, + pattern: string, + options?: GrepOptions +): Promise { + const key = normalize(path) + if (!docsKeyView.has(key)) { + throw new DocsCorpusError( + `Grep over the docs corpus must target a single page (e.g. path: "docs/workflows/blocks/agent.mdx"). "${path}" is not a docs page. Use glob("docs/**") to find the exact path, then grep that one page.` + ) + } + const page = await readDocsPage(key) + return grepReadResult(key, page, pattern, key, options) +} diff --git a/apps/sim/lib/copilot/docs/docs-path.test.ts b/apps/sim/lib/copilot/docs/docs-path.test.ts new file mode 100644 index 00000000000..c40ba0a7abb --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-path.test.ts @@ -0,0 +1,35 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { docsSourceCandidates, foldDocsIndexPath } from '@/lib/copilot/docs/docs-path' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' + +describe('foldDocsIndexPath', () => { + it('folds a section overview onto the section path', () => { + expect(foldDocsIndexPath('workflows/index.mdx')).toBe('workflows.mdx') + expect(foldDocsIndexPath('platform/enterprise/index.mdx')).toBe('platform/enterprise.mdx') + }) + + it('leaves a plain page untouched', () => { + expect(foldDocsIndexPath('workflows/blocks/agent.mdx')).toBe('workflows/blocks/agent.mdx') + expect(foldDocsIndexPath('agents.mdx')).toBe('agents.mdx') + }) + + it('does not fold a page merely named index', () => { + expect(foldDocsIndexPath('index.mdx')).toBe('index.mdx') + }) +}) + +describe('docsSourceCandidates', () => { + it('is the inverse of the fold — one candidate always reproduces the input', () => { + for (const publicPath of DOCS_MANIFEST) { + const candidates = docsSourceCandidates(publicPath) + expect(candidates.map(foldDocsIndexPath)).toContain(publicPath) + } + }) + + it('offers both on-disk layouts for a section path', () => { + expect(docsSourceCandidates('workflows.mdx')).toEqual(['workflows.mdx', 'workflows/index.mdx']) + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-path.ts b/apps/sim/lib/copilot/docs/docs-path.ts new file mode 100644 index 00000000000..b4e5ff66bc1 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-path.ts @@ -0,0 +1,58 @@ +/** + * The single definition of how a docs source file maps onto its public path. + * + * Fumadocs folds a section's `index.mdx` into the section URL itself, so + * `workflows/index.mdx` on disk is `/workflows` on the site (and + * `/workflows/index.mdx` is a 404). Three places need that rule — the manifest + * generator, the `source_document` -> VFS reverse mapping, and the vector + * search's scope filter — and hand-syncing it has bitten this repo before, so + * it lives here. + * + * Deliberately dependency-free: `scripts/sync-docs-manifest.ts` imports this by + * relative path, and it must not pull in the manifest it generates. + */ + +/** Suffix that marks a section overview page on disk. */ +export const DOCS_INDEX_SUFFIX = '/index.mdx' + +/** + * Top-level docs sections deliberately left out of the copilot's `docs/` tree. + * + * Two places must agree on this list or the corpus goes subtly wrong: the + * manifest generator (which decides what is readable) and the vector search's + * unscoped filter (which decides what is findable). If search still matched an + * unmounted section, every hit there would be a chunk the agent cannot then + * `read` — dropped as stale, silently shrinking the result set. + * + * Mounting a section later is not uniform work, so plan per section: + * - `academy` is plain mdx under `apps/docs/content/docs/en/academy` and is + * already indexed in `docs_embeddings` — removing it here and regenerating + * the manifest is the whole change. + * - `api-reference` is mostly generated from `apps/docs/openapi.json` at build + * time, so its pages have no source mdx for the generator to walk (only the + * four handwritten ones: authentication, getting-started, python, typescript). + * Mounting it properly needs the spec served publicly again — the + * `apps/docs/app/openapi.json` route existed for exactly this and was + * reverted — plus a generator branch that walks the spec's tags. + */ +export const UNMOUNTED_DOCS_SECTIONS = ['academy', 'api-reference'] as const + +/** + * Fold an `en`-relative mdx file path onto its public path — the value used as + * both the `docs/`-relative VFS path and the docs.sim.ai URL path. + */ +export function foldDocsIndexPath(mdxPath: string): string { + return mdxPath.endsWith(DOCS_INDEX_SUFFIX) + ? `${mdxPath.slice(0, -DOCS_INDEX_SUFFIX.length)}.mdx` + : mdxPath +} + +/** + * The inverse of {@link foldDocsIndexPath}: the on-disk file names a public + * path could have come from. A page is stored either as `.mdx` or, when + * it is a section overview, as `/index.mdx`. + */ +export function docsSourceCandidates(publicPath: string): [string, string] { + const stem = publicPath.replace(/\.mdx$/, '') + return [`${stem}.mdx`, `${stem}${DOCS_INDEX_SUFFIX}`] +} diff --git a/apps/sim/lib/copilot/docs/docs-search.test.ts b/apps/sim/lib/copilot/docs/docs-search.test.ts new file mode 100644 index 00000000000..5920c0d9fef --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-search.test.ts @@ -0,0 +1,293 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGenerateSearchEmbedding, capturedWhere, capturedLimit, mockRows } = vi.hoisted(() => ({ + mockGenerateSearchEmbedding: vi.fn(), + capturedWhere: { value: undefined as unknown }, + capturedLimit: { value: undefined as number | undefined }, + mockRows: { value: [] as unknown[] }, +})) + +vi.mock('@/lib/knowledge/embeddings', () => ({ + generateSearchEmbedding: mockGenerateSearchEmbedding, +})) + +/** + * Override the global drizzle mock with operators that record their arguments, + * so a test can assert on the `source_document` filter the scope produced. + */ +vi.mock('drizzle-orm', () => { + const op = + (name: string) => + (...args: unknown[]) => ({ op: name, args }) + return { + and: op('and'), + or: op('or'), + eq: op('eq'), + ne: op('ne'), + like: op('like'), + notLike: op('notLike'), + sql: (strings: TemplateStringsArray) => ({ op: 'sql', text: strings.join('?') }), + } +}) + +vi.mock('@sim/db', () => ({ + db: { + select: () => ({ + from: () => ({ + where: (condition: unknown) => { + capturedWhere.value = condition + return { + orderBy: () => ({ + limit: async (n: number) => { + capturedLimit.value = n + return mockRows.value + }, + }), + } + }, + }), + }), + }, +})) + +import { DocsSearchScopeError, searchDocs } from '@/lib/copilot/docs/docs-search' + +/** Render a drizzle condition to comparable SQL-ish text for assertions. */ +function whereText(): string { + return JSON.stringify(capturedWhere.value) +} + +describe('searchDocs path scoping', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockRows.value = [] + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('excludes unmounted sections when unscoped', async () => { + await searchDocs('cron') + expect(whereText()).toContain('academy/%') + expect(whereText()).toContain('api-reference/%') + }) + + it('treats a bare docs prefix as unscoped', async () => { + await searchDocs('cron', { path: 'docs/' }) + expect(whereText()).toContain('academy/%') + }) + + it('excludes the root homepage when unscoped — its chunks have no live docs/ path', async () => { + await searchDocs('cron') + expect(whereText()).toContain('"op":"ne"') + expect(whereText()).toContain('index.mdx') + }) + + it('scopes a page to both on-disk layouts', async () => { + await searchDocs('cron', { path: 'docs/workflows/blocks/agent.mdx' }) + const text = whereText() + expect(text).toContain('workflows/blocks/agent.mdx') + expect(text).toContain('workflows/blocks/agent/index.mdx') + }) + + it('maps a section overview page onto its index file', async () => { + await searchDocs('cron', { path: 'docs/workflows.mdx' }) + const text = whereText() + expect(text).toContain('workflows/index.mdx') + }) + + it('scopes a directory to its subtree', async () => { + await searchDocs('cron', { path: 'docs/workflows' }) + expect(whereText()).toContain('workflows/%') + }) + + it('includes a section overview stored in either on-disk layout', async () => { + await searchDocs('cron', { path: 'docs/workflows' }) + const text = whereText() + // `workflows/index.mdx` is inside the subtree; a sibling `workflows.mdx` is not, + // and fumadocs accepts either, so the scope must name it explicitly. + expect(text).toContain('workflows/%') + expect(text).toContain('workflows.mdx') + }) + + it('rejects a path outside the docs corpus', async () => { + await expect(searchDocs('cron', { path: 'files/report.pdf' })).rejects.toThrow( + DocsSearchScopeError + ) + }) + + it('rejects a docs path that is neither a page nor a section', async () => { + await expect(searchDocs('cron', { path: 'docs/not-a-real-section' })).rejects.toThrow( + /not a page or section/ + ) + }) + + it('rejects unmounted sections that exist on the site but not in the VFS', async () => { + await expect(searchDocs('cron', { path: 'docs/academy' })).rejects.toThrow( + /not a page or section/ + ) + }) +}) + +describe('searchDocs results', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('returns the docs/ path to read next, folding index pages', async () => { + mockRows.value = [ + { + chunkText: 'body', + sourceDocument: 'workflows/index.mdx', + sourceLink: 'https://docs.sim.ai/workflows', + headerText: 'Overview', + similarity: 0.8, + }, + ] + const { results } = await searchDocs('cron') + expect(results).toEqual([ + { + path: 'docs/workflows.mdx', + url: 'https://docs.sim.ai/workflows', + title: 'Overview', + content: 'body', + similarity: 0.8, + }, + ]) + }) + + it('drops chunks whose source has no live docs/ path', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'academy/lesson-1.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.9, + }, + { + chunkText: 'b', + sourceDocument: 'deleted-page.mdx', + sourceLink: 'y', + headerText: 'h', + similarity: 0.9, + }, + ] + expect((await searchDocs('cron')).results).toEqual([]) + }) + + it('returns the zero-candidate outcome without querying when the embedding is empty', async () => { + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [] }) + const outcome = await searchDocs('cron') + expect(outcome).toEqual({ + results: [], + candidatesConsidered: 0, + droppedBelowThreshold: 0, + droppedStale: 0, + }) + }) + + it('drops chunks below the similarity threshold', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.1, + }, + ] + expect((await searchDocs('cron')).results).toEqual([]) + }) +}) + +describe('searchDocs shortfall reporting', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('counts why candidates were dropped so an empty set is explainable', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.1, + }, + { + chunkText: 'b', + sourceDocument: 'deleted-page.mdx', + sourceLink: 'y', + headerText: 'h', + similarity: 0.9, + }, + ] + const outcome = await searchDocs('cron') + expect(outcome).toEqual({ + results: [], + candidatesConsidered: 2, + droppedBelowThreshold: 1, + droppedStale: 1, + }) + }) + + it('reports no drops when every candidate survives', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.9, + }, + ] + const outcome = await searchDocs('cron') + expect(outcome.droppedBelowThreshold).toBe(0) + expect(outcome.droppedStale).toBe(0) + expect(outcome.results).toHaveLength(1) + }) +}) + +describe('searchDocs topK clamping', () => { + beforeEach(() => { + capturedLimit.value = undefined + mockRows.value = [] + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('defaults to 5 when unspecified', async () => { + await searchDocs('cron') + expect(capturedLimit.value).toBe(5) + }) + + it('caps at 25 — the documented max, which the old tool never enforced', async () => { + await searchDocs('cron', { topK: 500 }) + expect(capturedLimit.value).toBe(25) + }) + + it('floors at 1', async () => { + await searchDocs('cron', { topK: 0 }) + expect(capturedLimit.value).toBe(1) + await searchDocs('cron', { topK: -8 }) + expect(capturedLimit.value).toBe(1) + }) + + it('truncates a fractional count', async () => { + await searchDocs('cron', { topK: 7.9 }) + expect(capturedLimit.value).toBe(7) + }) + + it('falls back to the default rather than passing NaN to the query', async () => { + // Math.min/Math.max propagate NaN, so a bare clamp would reach `.limit(NaN)`. + await searchDocs('cron', { topK: Number.NaN }) + expect(capturedLimit.value).toBe(5) + await searchDocs('cron', { topK: 'twelve' as unknown as number }) + expect(capturedLimit.value).toBe(5) + await searchDocs('cron', { topK: Number.POSITIVE_INFINITY }) + expect(capturedLimit.value).toBe(5) + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-search.ts b/apps/sim/lib/copilot/docs/docs-search.ts new file mode 100644 index 00000000000..0b70d840608 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-search.ts @@ -0,0 +1,205 @@ +import { db } from '@sim/db' +import { docsEmbeddings } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { and, eq, like, ne, notLike, or, sql } from 'drizzle-orm' +import { docsPathForSourceDocument, isDocsDir, isDocsPage } from '@/lib/copilot/docs/docs-corpus' +import { docsSourceCandidates, UNMOUNTED_DOCS_SECTIONS } from '@/lib/copilot/docs/docs-path' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' + +const logger = createLogger('DocsSearch') + +const SIMILARITY_THRESHOLD = 0.3 +const DEFAULT_TOP_K = 5 +const MAX_TOP_K = 25 + +export interface DocsSearchResult { + /** The `docs/` VFS path this chunk came from — pass it to `read` for the full page. */ + path: string + /** Public docs.sim.ai URL for the section, for citation. */ + url: string + title: string + content: string + similarity: number +} + +/** + * A search result set plus why it may be shorter than `topK`. The SQL LIMIT is + * applied before the threshold and liveness filters, so these counts are what + * distinguishes "nothing matched" from "matches were filtered out". + */ +export interface DocsSearchOutcome { + results: DocsSearchResult[] + /** Rows the vector search returned before filtering. */ + candidatesConsidered: number + /** Candidates dropped for scoring below the similarity threshold. */ + droppedBelowThreshold: number + /** Candidates dropped because their page is no longer in the docs manifest. */ + droppedStale: number +} + +/** + * Thrown when the caller scopes a search to a `path` that is not a real page or + * section in the docs corpus. Surfaced verbatim so the model can correct itself + * rather than reading an empty result as "the docs say nothing about this". + */ +export class DocsSearchScopeError extends Error { + readonly code = 'DOCS_SEARCH_SCOPE' as const + constructor(message: string) { + super(message) + this.name = 'DocsSearchScopeError' + } +} + +/** + * Translate an optional `docs/` VFS path into a `source_document` filter. + * + * `source_document` stores the en-relative mdx file path, while VFS paths mirror + * the public URL — so a section overview is `docs/workflows.mdx` in the VFS but + * `workflows/index.mdx` (or `workflows.mdx`) on disk. A directory scope covers + * the whole subtree plus the overview in either layout. + * + * An unscoped search excludes every {@link UNMOUNTED_DOCS_SECTIONS} section: + * they are indexed but not mounted in the VFS, so a hit there would be a chunk + * the agent cannot then read. The root homepage (`index.mdx`) is excluded for + * the same reason — the manifest generator drops it (its URL is `/`, which + * redirects), so its chunks would only ever be counted against topK and then + * discarded as stale. + */ +function scopeCondition(path?: string) { + const normalized = (path ?? '').trim().replace(/^\/+/, '').replace(/\/+$/, '') + if (normalized === '' || normalized === 'docs') { + return and( + ne(docsEmbeddings.sourceDocument, 'index.mdx'), + ...UNMOUNTED_DOCS_SECTIONS.map((section) => + notLike(docsEmbeddings.sourceDocument, `${section}/%`) + ) + ) + } + + if (!normalized.startsWith('docs/')) { + throw new DocsSearchScopeError( + `path must be a docs/ VFS path (got "${path}"). Use glob("docs/**") to find one, or omit path to search everything.` + ) + } + + const tail = normalized.slice('docs/'.length) + + if (isDocsPage(normalized)) { + // One page: on disk it is either `.mdx` or `/index.mdx`. + const [pageFile, indexFile] = docsSourceCandidates(tail) + return or( + eq(docsEmbeddings.sourceDocument, pageFile), + eq(docsEmbeddings.sourceDocument, indexFile) + ) + } + + if (isDocsDir(normalized)) { + // Everything under the directory, PLUS a sibling `.mdx`. Fumadocs + // accepts either layout for a section overview and only `/index.mdx` + // is inside the subtree, so matching the prefix alone would silently omit + // the overview for the sibling layout — page scope already covers both. + return or( + like(docsEmbeddings.sourceDocument, `${escapeLikePattern(tail)}/%`), + eq(docsEmbeddings.sourceDocument, `${tail}.mdx`) + ) + } + + throw new DocsSearchScopeError( + `"${path}" is not a page or section in the docs corpus. Use glob("docs/**") to find a valid path, or omit path to search everything.` + ) +} + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, (char) => `\\${char}`) +} + +/** + * Clamp a caller-supplied result count into [1, {@link MAX_TOP_K}]. + * + * Guards magnitude AND type: `Math.min`/`Math.max` propagate NaN, so a + * non-numeric value would otherwise reach the query as `.limit(NaN)`. The + * generated tool schema rejects a non-number upstream today, but this function + * is also called directly, so it does not rely on that. + */ +function clampTopK(requested: number | undefined): number { + if (requested === undefined || !Number.isFinite(requested)) return DEFAULT_TOP_K + return Math.min(Math.max(Math.trunc(requested), 1), MAX_TOP_K) +} + +/** + * Semantic search over the indexed docs corpus (`docs_embeddings`, rebuilt by + * `scripts/process-docs.ts` on release). Every result carries the `docs/` path + * it came from so the caller can `read` the full page next. + * + * The index lags the VFS: a page added since the last index rebuild is readable + * but not searchable, and a deleted one can still return chunks. Results whose + * source no longer maps to a live `docs/` path are dropped. + * + * Because those drops happen after the SQL LIMIT, a caller can get fewer hits + * than it asked for — or none at all when every candidate was filtered. The + * returned {@link DocsSearchOutcome} reports that explicitly so an empty result + * is never mistaken for "the documentation does not cover this". + */ +export async function searchDocs( + query: string, + options?: { path?: string; topK?: number } +): Promise { + if (!query || typeof query !== 'string') throw new Error('query is required') + + const topK = clampTopK(options?.topK) + const where = scopeCondition(options?.path) + + logger.info('Executing docs search', { query, topK, path: options?.path ?? null }) + + const { embedding: queryEmbedding } = await generateSearchEmbedding(query) + if (!queryEmbedding || queryEmbedding.length === 0) { + return { results: [], candidatesConsidered: 0, droppedBelowThreshold: 0, droppedStale: 0 } + } + + const rows = await db + .select({ + chunkText: docsEmbeddings.chunkText, + sourceDocument: docsEmbeddings.sourceDocument, + sourceLink: docsEmbeddings.sourceLink, + headerText: docsEmbeddings.headerText, + similarity: sql`1 - (${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector)`, + }) + .from(docsEmbeddings) + .where(where) + .orderBy(sql`${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector`) + .limit(topK) + + const results: DocsSearchResult[] = [] + let droppedBelowThreshold = 0 + let droppedStale = 0 + for (const row of rows) { + if (row.similarity < SIMILARITY_THRESHOLD) { + droppedBelowThreshold++ + continue + } + const path = docsPathForSourceDocument(row.sourceDocument) + if (!path) { + droppedStale++ + continue + } + results.push({ + path, + url: String(row.sourceLink || '#'), + title: String(row.headerText || 'Untitled Section'), + content: String(row.chunkText || ''), + similarity: row.similarity, + }) + } + + logger.info('Docs search complete', { + count: results.length, + droppedBelowThreshold, + droppedStale, + }) + return { + results, + candidatesConsidered: rows.length, + droppedBelowThreshold, + droppedStale, + } +} diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts new file mode 100644 index 00000000000..77a5968fd76 --- /dev/null +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -0,0 +1,368 @@ +// AUTO-GENERATED FILE. DO NOT EDIT. +// Generated from apps/docs/content/docs/en by scripts/sync-docs-manifest.ts +// Run: bun run docs-manifest:generate +// + +/** + * Every page in the copilot's read-only `docs/` VFS tree, as a path that is + * simultaneously the `docs/`-relative VFS path and the docs.sim.ai URL path + * (so `docs/workflows/blocks/agent.mdx` reads + * `https://docs.sim.ai/workflows/blocks/agent.mdx`). Sorted. + */ +export const DOCS_MANIFEST: readonly string[] = [ + 'agents.mdx', + 'agents/choosing.mdx', + 'agents/custom-tools.mdx', + 'agents/mcp.mdx', + 'agents/skills.mdx', + 'files.mdx', + 'files/editor.mdx', + 'files/generating.mdx', + 'files/passing-files.mdx', + 'files/using-in-workflows.mdx', + 'getting-started.mdx', + 'integrations.mdx', + 'integrations/a2a.mdx', + 'integrations/agentmail.mdx', + 'integrations/agentphone.mdx', + 'integrations/agiloft.mdx', + 'integrations/ahrefs.mdx', + 'integrations/airtable-service-account.mdx', + 'integrations/airtable.mdx', + 'integrations/airweave.mdx', + 'integrations/algolia.mdx', + 'integrations/amplitude.mdx', + 'integrations/apify.mdx', + 'integrations/apollo.mdx', + 'integrations/appconfig.mdx', + 'integrations/arxiv.mdx', + 'integrations/asana-service-account.mdx', + 'integrations/asana.mdx', + 'integrations/ashby.mdx', + 'integrations/athena.mdx', + 'integrations/atlassian-service-account.mdx', + 'integrations/attio-service-account.mdx', + 'integrations/attio.mdx', + 'integrations/azure_devops.mdx', + 'integrations/box-service-account.mdx', + 'integrations/box.mdx', + 'integrations/brandfetch.mdx', + 'integrations/brex.mdx', + 'integrations/brightdata.mdx', + 'integrations/browser_use.mdx', + 'integrations/buffer.mdx', + 'integrations/calcom-service-account.mdx', + 'integrations/calcom.mdx', + 'integrations/calendly.mdx', + 'integrations/circleback.mdx', + 'integrations/clay.mdx', + 'integrations/clerk.mdx', + 'integrations/clickhouse.mdx', + 'integrations/clickup-service-account.mdx', + 'integrations/clickup.mdx', + 'integrations/cloudflare.mdx', + 'integrations/cloudformation.mdx', + 'integrations/cloudwatch.mdx', + 'integrations/codepipeline.mdx', + 'integrations/confluence.mdx', + 'integrations/context_dev.mdx', + 'integrations/convex.mdx', + 'integrations/crowdstrike.mdx', + 'integrations/cursor.mdx', + 'integrations/dagster.mdx', + 'integrations/databricks.mdx', + 'integrations/datadog.mdx', + 'integrations/datagma.mdx', + 'integrations/daytona.mdx', + 'integrations/deployments.mdx', + 'integrations/devin.mdx', + 'integrations/discord.mdx', + 'integrations/docusign.mdx', + 'integrations/downdetector.mdx', + 'integrations/dropbox.mdx', + 'integrations/dropcontact.mdx', + 'integrations/dspy.mdx', + 'integrations/dub.mdx', + 'integrations/duckduckgo.mdx', + 'integrations/dynamodb.mdx', + 'integrations/elasticsearch.mdx', + 'integrations/elevenlabs.mdx', + 'integrations/emailbison.mdx', + 'integrations/enrich.mdx', + 'integrations/enrichment.mdx', + 'integrations/enrow.mdx', + 'integrations/evernote.mdx', + 'integrations/exa.mdx', + 'integrations/extend.mdx', + 'integrations/fathom.mdx', + 'integrations/file.mdx', + 'integrations/findymail.mdx', + 'integrations/firecrawl.mdx', + 'integrations/fireflies.mdx', + 'integrations/flint.mdx', + 'integrations/gamma.mdx', + 'integrations/github.mdx', + 'integrations/gitlab.mdx', + 'integrations/gmail.mdx', + 'integrations/gong.mdx', + 'integrations/google-service-account.mdx', + 'integrations/google_ads.mdx', + 'integrations/google_appsheet.mdx', + 'integrations/google_bigquery.mdx', + 'integrations/google_books.mdx', + 'integrations/google_calendar.mdx', + 'integrations/google_contacts.mdx', + 'integrations/google_docs.mdx', + 'integrations/google_drive.mdx', + 'integrations/google_forms.mdx', + 'integrations/google_groups.mdx', + 'integrations/google_maps.mdx', + 'integrations/google_meet.mdx', + 'integrations/google_pagespeed.mdx', + 'integrations/google_search.mdx', + 'integrations/google_sheets.mdx', + 'integrations/google_slides.mdx', + 'integrations/google_tasks.mdx', + 'integrations/google_translate.mdx', + 'integrations/google_vault.mdx', + 'integrations/grafana.mdx', + 'integrations/grain.mdx', + 'integrations/granola.mdx', + 'integrations/greenhouse.mdx', + 'integrations/greptile.mdx', + 'integrations/hex.mdx', + 'integrations/hubspot-service-account.mdx', + 'integrations/hubspot-setup.mdx', + 'integrations/hubspot.mdx', + 'integrations/huggingface.mdx', + 'integrations/hunter.mdx', + 'integrations/iam.mdx', + 'integrations/icypeas.mdx', + 'integrations/identity_center.mdx', + 'integrations/imap.mdx', + 'integrations/incidentio.mdx', + 'integrations/infisical.mdx', + 'integrations/instantly.mdx', + 'integrations/intercom.mdx', + 'integrations/jina.mdx', + 'integrations/jira.mdx', + 'integrations/jira_service_management.mdx', + 'integrations/jupyter.mdx', + 'integrations/kalshi.mdx', + 'integrations/ketch.mdx', + 'integrations/knowledge.mdx', + 'integrations/langsmith.mdx', + 'integrations/latex.mdx', + 'integrations/launchdarkly.mdx', + 'integrations/leadmagic.mdx', + 'integrations/lemlist.mdx', + 'integrations/linear-service-account.mdx', + 'integrations/linear.mdx', + 'integrations/linkedin.mdx', + 'integrations/linkup.mdx', + 'integrations/linq.mdx', + 'integrations/logs.mdx', + 'integrations/loops.mdx', + 'integrations/luma.mdx', + 'integrations/mailchimp.mdx', + 'integrations/mailgun.mdx', + 'integrations/managed_agent.mdx', + 'integrations/mem0.mdx', + 'integrations/memory.mdx', + 'integrations/microsoft_ad.mdx', + 'integrations/microsoft_dataverse.mdx', + 'integrations/microsoft_excel.mdx', + 'integrations/microsoft_planner.mdx', + 'integrations/microsoft_teams.mdx', + 'integrations/millionverifier.mdx', + 'integrations/mistral_parse.mdx', + 'integrations/monday-service-account.mdx', + 'integrations/monday.mdx', + 'integrations/mongodb.mdx', + 'integrations/mysql.mdx', + 'integrations/neo4j.mdx', + 'integrations/neverbounce.mdx', + 'integrations/new_relic.mdx', + 'integrations/notion-service-account.mdx', + 'integrations/notion.mdx', + 'integrations/obsidian.mdx', + 'integrations/okta.mdx', + 'integrations/onedrive.mdx', + 'integrations/onepassword.mdx', + 'integrations/openai.mdx', + 'integrations/outlook.mdx', + 'integrations/pagerduty.mdx', + 'integrations/parallel_ai.mdx', + 'integrations/peopledatalabs.mdx', + 'integrations/perplexity.mdx', + 'integrations/persona.mdx', + 'integrations/pinecone.mdx', + 'integrations/pipedrive-service-account.mdx', + 'integrations/pipedrive.mdx', + 'integrations/polymarket.mdx', + 'integrations/postgresql.mdx', + 'integrations/posthog.mdx', + 'integrations/profound.mdx', + 'integrations/prospeo.mdx', + 'integrations/pulse.mdx', + 'integrations/qdrant.mdx', + 'integrations/quartr.mdx', + 'integrations/quiver.mdx', + 'integrations/railway.mdx', + 'integrations/rb2b.mdx', + 'integrations/rds.mdx', + 'integrations/reddit.mdx', + 'integrations/redis.mdx', + 'integrations/reducto.mdx', + 'integrations/resend.mdx', + 'integrations/revenuecat.mdx', + 'integrations/rippling.mdx', + 'integrations/rocketlane.mdx', + 'integrations/rootly.mdx', + 'integrations/s3.mdx', + 'integrations/salesforce-service-account.mdx', + 'integrations/salesforce.mdx', + 'integrations/sap_concur.mdx', + 'integrations/sap_s4hana.mdx', + 'integrations/secrets_manager.mdx', + 'integrations/sendblue.mdx', + 'integrations/sendgrid.mdx', + 'integrations/sentry.mdx', + 'integrations/serper.mdx', + 'integrations/servicenow.mdx', + 'integrations/ses.mdx', + 'integrations/sftp.mdx', + 'integrations/sharepoint.mdx', + 'integrations/shopify-service-account.mdx', + 'integrations/shopify.mdx', + 'integrations/similarweb.mdx', + 'integrations/sixtyfour.mdx', + 'integrations/slack.mdx', + 'integrations/smtp.mdx', + 'integrations/sportmonks.mdx', + 'integrations/sqs.mdx', + 'integrations/square.mdx', + 'integrations/ssh.mdx', + 'integrations/stagehand.mdx', + 'integrations/stripe.mdx', + 'integrations/sts.mdx', + 'integrations/supabase.mdx', + 'integrations/table.mdx', + 'integrations/tailscale.mdx', + 'integrations/tavily.mdx', + 'integrations/telegram.mdx', + 'integrations/temporal.mdx', + 'integrations/textract.mdx', + 'integrations/thrive.mdx', + 'integrations/tiktok.mdx', + 'integrations/tinybird.mdx', + 'integrations/trello-service-account.mdx', + 'integrations/trello.mdx', + 'integrations/trigger_dev.mdx', + 'integrations/twilio.mdx', + 'integrations/twilio_sms.mdx', + 'integrations/twilio_voice.mdx', + 'integrations/typeform.mdx', + 'integrations/upstash.mdx', + 'integrations/uptimerobot.mdx', + 'integrations/vanta.mdx', + 'integrations/vercel.mdx', + 'integrations/wealthbox-service-account.mdx', + 'integrations/wealthbox.mdx', + 'integrations/webflow-service-account.mdx', + 'integrations/webflow.mdx', + 'integrations/whatsapp.mdx', + 'integrations/wikipedia.mdx', + 'integrations/wiza.mdx', + 'integrations/wordpress.mdx', + 'integrations/workday.mdx', + 'integrations/x.mdx', + 'integrations/youtube.mdx', + 'integrations/zendesk.mdx', + 'integrations/zep.mdx', + 'integrations/zerobounce.mdx', + 'integrations/zoom-service-account.mdx', + 'integrations/zoom.mdx', + 'integrations/zoominfo.mdx', + 'introduction.mdx', + 'keyboard-shortcuts.mdx', + 'knowledgebase.mdx', + 'knowledgebase/chunking-strategies.mdx', + 'knowledgebase/connectors.mdx', + 'knowledgebase/debugging-retrieval.mdx', + 'knowledgebase/tags.mdx', + 'knowledgebase/using-in-workflows.mdx', + 'logs-debugging.mdx', + 'logs-debugging/alerts.mdx', + 'logs-debugging/logging.mdx', + 'mothership.mdx', + 'mothership/files.mdx', + 'mothership/knowledge.mdx', + 'mothership/mailer.mdx', + 'mothership/research.mdx', + 'mothership/tables.mdx', + 'mothership/tasks.mdx', + 'mothership/workflows.mdx', + 'platform/costs.mdx', + 'platform/credentials.mdx', + 'platform/enterprise.mdx', + 'platform/enterprise/access-control.mdx', + 'platform/enterprise/audit-logs.mdx', + 'platform/enterprise/custom-blocks.mdx', + 'platform/enterprise/data-drains.mdx', + 'platform/enterprise/data-retention.mdx', + 'platform/enterprise/forks.mdx', + 'platform/enterprise/self-hosted.mdx', + 'platform/enterprise/session-policies.mdx', + 'platform/enterprise/sso.mdx', + 'platform/enterprise/verified-domains.mdx', + 'platform/enterprise/whitelabeling.mdx', + 'platform/organization.mdx', + 'platform/permissions.mdx', + 'platform/self-hosting.mdx', + 'platform/self-hosting/docker.mdx', + 'platform/self-hosting/environment-variables.mdx', + 'platform/self-hosting/kubernetes.mdx', + 'platform/self-hosting/object-storage.mdx', + 'platform/self-hosting/platforms.mdx', + 'platform/self-hosting/troubleshooting.mdx', + 'platform/workspaces.mdx', + 'quick-reference.mdx', + 'tables.mdx', + 'tables/using-in-workflows.mdx', + 'tables/workflow-columns.mdx', + 'workflows.mdx', + 'workflows/blocks/agent.mdx', + 'workflows/blocks/api.mdx', + 'workflows/blocks/condition.mdx', + 'workflows/blocks/credential.mdx', + 'workflows/blocks/evaluator.mdx', + 'workflows/blocks/function.mdx', + 'workflows/blocks/guardrails.mdx', + 'workflows/blocks/human-in-the-loop.mdx', + 'workflows/blocks/logs.mdx', + 'workflows/blocks/loop.mdx', + 'workflows/blocks/parallel.mdx', + 'workflows/blocks/pi.mdx', + 'workflows/blocks/response.mdx', + 'workflows/blocks/router.mdx', + 'workflows/blocks/variables.mdx', + 'workflows/blocks/wait.mdx', + 'workflows/blocks/webhook.mdx', + 'workflows/blocks/workflow.mdx', + 'workflows/connections.mdx', + 'workflows/data-flow.mdx', + 'workflows/deployment.mdx', + 'workflows/deployment/agent-events.mdx', + 'workflows/deployment/api.mdx', + 'workflows/deployment/chat.mdx', + 'workflows/deployment/mcp.mdx', + 'workflows/how-it-runs.mdx', + 'workflows/triggers/rss.mdx', + 'workflows/triggers/schedule.mdx', + 'workflows/triggers/sim.mdx', + 'workflows/triggers/start.mdx', + 'workflows/triggers/table.mdx', + 'workflows/triggers/webhook.mdx', + 'workflows/variables.mdx', +] diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 88159f0d49b..56a6dca6692 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -62,7 +62,6 @@ export interface ToolCatalogEntry { | 'get_deployed_workflow_state' | 'get_deployment_log' | 'get_page_contents' - | 'get_platform_actions' | 'get_scheduled_task_logs' | 'get_workflow_data' | 'get_workflow_run_options' @@ -105,7 +104,7 @@ export interface ToolCatalogEntry { | 'scheduled_task' | 'scrape_page' | 'search' - | 'search_documentation' + | 'search_docs' | 'search_integration_tools' | 'search_knowledge_base' | 'search_library_docs' @@ -182,7 +181,6 @@ export interface ToolCatalogEntry { | 'get_deployed_workflow_state' | 'get_deployment_log' | 'get_page_contents' - | 'get_platform_actions' | 'get_scheduled_task_logs' | 'get_workflow_data' | 'get_workflow_run_options' @@ -225,7 +223,7 @@ export interface ToolCatalogEntry { | 'scheduled_task' | 'scrape_page' | 'search' - | 'search_documentation' + | 'search_docs' | 'search_integration_tools' | 'search_knowledge_base' | 'search_library_docs' @@ -2476,14 +2474,6 @@ export const GetPageContents: ToolCatalogEntry = { }, } -export const GetPlatformActions: ToolCatalogEntry = { - id: 'get_platform_actions', - name: 'get_platform_actions', - route: 'sim', - mode: 'async', - parameters: { type: 'object', properties: {} }, -} - export const GetScheduledTaskLogs: ToolCatalogEntry = { id: 'get_scheduled_task_logs', name: 'get_scheduled_task_logs', @@ -4023,7 +4013,7 @@ export const Search: ToolCatalogEntry = { properties: { task: { description: - "One short scoping sentence — the search agent has full conversation context. Example: 'find current Stripe metered-billing API limits' or 'count how many rows in the leads table have invalid emails'.", + "A fully self-contained task — the search agent sees none of this conversation, so include the question plus every name, id, constraint, and prior finding it needs. Example: 'find current Stripe metered-billing API limits' or 'count how many rows in the leads table have invalid emails'.", type: 'string', }, }, @@ -4034,21 +4024,21 @@ export const Search: ToolCatalogEntry = { internal: true, } -export const SearchDocumentation: ToolCatalogEntry = { - id: 'search_documentation', - name: 'search_documentation', +export const SearchDocs: ToolCatalogEntry = { + id: 'search_docs', + name: 'search_docs', route: 'sim', mode: 'async', parameters: { type: 'object', properties: { - query: { type: 'string', description: 'The search query' }, - topK: { - type: 'number', + path: { + type: 'string', description: - 'Number of results to return (default 10). Not clamped — keep it small, since each result is a full doc chunk.', - default: 10, + 'Optional docs/ VFS path (a page such as docs/workflows/blocks/agent.mdx, or a section such as docs/workflows) that limits the search scope', }, + query: { type: 'string', description: 'The search query' }, + topK: { type: 'number', description: 'Number of results (default 5, max 25)' }, }, required: ['query'], }, @@ -4735,7 +4725,7 @@ export const UserTable: ToolCatalogEntry = { newType: { type: 'string', description: - "New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn't match one of them.", + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', }, offset: { type: 'number', @@ -4744,7 +4734,7 @@ export const UserTable: ToolCatalogEntry = { options: { type: 'array', description: - 'Choices for a select (enum) column, as a list of display names, e.g. ["Open", "Closed"]. Required when creating or converting to a select column. On update_column this REPLACES the option list: options kept by name keep their cells, and cells holding a removed option are cleared. Max 100.', + 'Choices for a select (enum) column, as a list of display names, e.g. ["Open", "Closed"]. Required when creating or converting to a select column. On update_column this REPLACES the option list and is matched against the current one BY NAME: a name still present keeps its cells, a name no longer present is removed and cleared from every cell that held it. Send the full list including the options you are keeping — omitting one deletes it. There is no in-place rename, so re-sending an option under a new name clears the cells that held the old one. Max 100.', items: { type: 'string' }, }, outputColumnNames: { @@ -5462,7 +5452,6 @@ export const TOOL_CATALOG: Record = { [GetDeployedWorkflowState.id]: GetDeployedWorkflowState, [GetDeploymentLog.id]: GetDeploymentLog, [GetPageContents.id]: GetPageContents, - [GetPlatformActions.id]: GetPlatformActions, [GetScheduledTaskLogs.id]: GetScheduledTaskLogs, [GetWorkflowData.id]: GetWorkflowData, [GetWorkflowRunOptions.id]: GetWorkflowRunOptions, @@ -5505,7 +5494,7 @@ export const TOOL_CATALOG: Record = { [ScheduledTask.id]: ScheduledTask, [ScrapePage.id]: ScrapePage, [Search.id]: Search, - [SearchDocumentation.id]: SearchDocumentation, + [SearchDocs.id]: SearchDocs, [SearchIntegrationTools.id]: SearchIntegrationTools, [SearchKnowledgeBase.id]: SearchKnowledgeBase, [SearchLibraryDocs.id]: SearchLibraryDocs, diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index e527c57b544..0314f9841b8 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -2167,13 +2167,6 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_platform_actions: { - parameters: { - type: 'object', - properties: {}, - }, - resultSchema: undefined, - }, get_scheduled_task_logs: { parameters: { type: 'object', @@ -3673,7 +3666,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { properties: { task: { description: - "One short scoping sentence — the search agent has full conversation context. Example: 'find current Stripe metered-billing API limits' or 'count how many rows in the leads table have invalid emails'.", + "A fully self-contained task — the search agent sees none of this conversation, so include the question plus every name, id, constraint, and prior finding it needs. Example: 'find current Stripe metered-billing API limits' or 'count how many rows in the leads table have invalid emails'.", type: 'string', }, }, @@ -3682,19 +3675,22 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_documentation: { + search_docs: { parameters: { type: 'object', properties: { + path: { + type: 'string', + description: + 'Optional docs/ VFS path (a page such as docs/workflows/blocks/agent.mdx, or a section such as docs/workflows) that limits the search scope', + }, query: { type: 'string', description: 'The search query', }, topK: { type: 'number', - description: - 'Number of results to return (default 10). Not clamped — keep it small, since each result is a full doc chunk.', - default: 10, + description: 'Number of results (default 5, max 25)', }, }, required: ['query'], @@ -4386,7 +4382,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - "New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn't match one of them.", + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', }, offset: { type: 'number', @@ -4395,7 +4391,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { options: { type: 'array', description: - 'Choices for a select (enum) column, as a list of display names, e.g. ["Open", "Closed"]. Required when creating or converting to a select column. On update_column this REPLACES the option list: options kept by name keep their cells, and cells holding a removed option are cleared. Max 100.', + 'Choices for a select (enum) column, as a list of display names, e.g. ["Open", "Closed"]. Required when creating or converting to a select column. On update_column this REPLACES the option list and is matched against the current one BY NAME: a name still present keeps its cells, a name no longer present is removed and cleared from every cell that held it. Send the full list including the options you are keeping — omitting one deletes it. There is no in-place rename, so re-sending an option under a new name clears the cells that held the old one. Max 100.', items: { type: 'string', }, diff --git a/apps/sim/lib/copilot/tool-executor/register-handlers.ts b/apps/sim/lib/copilot/tool-executor/register-handlers.ts index 29e18387f7c..3f066edea9e 100644 --- a/apps/sim/lib/copilot/tool-executor/register-handlers.ts +++ b/apps/sim/lib/copilot/tool-executor/register-handlers.ts @@ -17,7 +17,6 @@ import { GetBlockUpstreamReferences, GetDeployedWorkflowState, GetDeploymentLog, - GetPlatformActions, GetWorkflowData, GetWorkflowRunOptions, Glob as GlobTool, @@ -87,7 +86,6 @@ import { executeManageMcpTool } from '../tools/handlers/management/manage-mcp-to import { executeManageSkill } from '../tools/handlers/management/manage-skill' import { executeMaterializeFile } from '../tools/handlers/materialize-file' import { executeOAuthGetAuthLink, executeOAuthRequestAccess } from '../tools/handlers/oauth' -import { executeGetPlatformActions } from '../tools/handlers/platform' import { executeOpenResource } from '../tools/handlers/resources' import { executeRestoreResource } from '../tools/handlers/restore-resource' import { executeRunCode } from '../tools/handlers/run-code' @@ -198,7 +196,6 @@ function buildHandlerMap(): Record { [OauthRequestAccess.id]: h(executeOAuthRequestAccess), [OpenResource.id]: h(executeOpenResource), [RestoreResource.id]: h(executeRestoreResource), - [GetPlatformActions.id]: h(executeGetPlatformActions), [ListIntegrationTools.id]: h(executeListIntegrationTools), [MaterializeFile.id]: h(executeMaterializeFile), [FunctionExecute.id]: h(executeFunctionExecute), diff --git a/apps/sim/lib/copilot/tools/client/hidden-tools.ts b/apps/sim/lib/copilot/tools/client/hidden-tools.ts index 7609e0e570b..09b055ce3b2 100644 --- a/apps/sim/lib/copilot/tools/client/hidden-tools.ts +++ b/apps/sim/lib/copilot/tools/client/hidden-tools.ts @@ -6,6 +6,11 @@ // user-meaningful action, only the resolved call_integration_tool row is. // load_skill is the same shape — the agent pulling in a reference guide before // doing the work is a step toward the action, not the action. +// search_documentation (the pre-rename id of search_docs) and +// get_platform_actions (retired; the quick reference lives in the docs corpus) +// are fully unregistered server-side, but historical persisted chats still +// contain their tool calls — like load_agent_skill, these entries stay forever +// so replaying an old transcript never renders a chip for a retired tool. const HIDDEN_TOOL_NAMES = new Set([ 'load_agent_skill', 'load_custom_tool', @@ -13,6 +18,8 @@ const HIDDEN_TOOL_NAMES = new Set([ 'load_integration_tool', 'load_skill', 'search_integration_tools', + 'search_documentation', + 'get_platform_actions', ]) export function isToolHiddenInUi(toolName: string | undefined): boolean { diff --git a/apps/sim/lib/copilot/tools/client/store-utils.test.ts b/apps/sim/lib/copilot/tools/client/store-utils.test.ts index 7a849821895..fa6de4c0b14 100644 --- a/apps/sim/lib/copilot/tools/client/store-utils.test.ts +++ b/apps/sim/lib/copilot/tools/client/store-utils.test.ts @@ -49,6 +49,26 @@ describe('resolveToolDisplay', () => { ).toBe('Read RET XYZ') }) + it('formats docs corpus reads as Section/page', () => { + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.success, { + path: 'docs/workflows/blocks/agent.mdx', + })?.text + ).toBe('Read Workflows/agent') + + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.executing, { + path: 'docs/integrations/gmail.mdx', + })?.text + ).toBe('Reading Integrations/gmail') + + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.error, { + path: 'docs/getting-started.mdx', + })?.text + ).toBe('Attempted to read Getting-started') + }) + it('decodes percent-encoded VFS path segments for display', () => { expect( resolveToolDisplay(ReadTool.id, ClientToolCallState.executing, { diff --git a/apps/sim/lib/copilot/tools/client/store-utils.ts b/apps/sim/lib/copilot/tools/client/store-utils.ts index 92bbeb6c572..4260f17823a 100644 --- a/apps/sim/lib/copilot/tools/client/store-utils.ts +++ b/apps/sim/lib/copilot/tools/client/store-utils.ts @@ -97,6 +97,10 @@ function describeReadTarget(path: string | undefined): string | undefined { if (segments.length === 0) return undefined + if (segments[0] === 'docs') { + return describeDocsReadTarget(segments) + } + const resourceType = VFS_DIR_TO_RESOURCE[segments[0]] if (!resourceType) { return humanizeDisplayIdentifier(stripExtension(segments[segments.length - 1]), 'sentence') @@ -140,6 +144,23 @@ function describeFileReadTarget(segments: string[]): string { return lastSegment } +/** + * Labels a docs/ corpus read as `
/` (e.g. `Workflows/agent` for + * docs/workflows/blocks/agent.mdx). Top-level pages show just their capitalized + * name (e.g. `Getting-started` for docs/getting-started.mdx). + */ +function describeDocsReadTarget(segments: string[]): string { + const rest = segments.slice(1) + if (rest.length === 0) return 'docs' + const leaf = stripExtension(rest[rest.length - 1]) + if (rest.length === 1) return capitalizeFirst(leaf) + return `${capitalizeFirst(rest[0])}/${leaf}` +} + +function capitalizeFirst(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1) +} + function getLeafResourceSegment(segments: string[]): string { const lastSegment = segments[segments.length - 1] || '' if (hasFileExtension(lastSegment) && segments.length > 1) { diff --git a/apps/sim/lib/copilot/tools/handlers/platform-actions.ts b/apps/sim/lib/copilot/tools/handlers/platform-actions.ts deleted file mode 100644 index c3c3ac14384..00000000000 --- a/apps/sim/lib/copilot/tools/handlers/platform-actions.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Static content for the get_platform_actions tool. - * Contains the Sim platform quick reference and keyboard shortcuts. - */ -export const PLATFORM_ACTIONS_CONTENT = `# Sim Platform Quick Reference & Keyboard Shortcuts - -## Keyboard Shortcuts -**Mod** = Cmd (macOS) / Ctrl (Windows/Linux). Shortcuts work when canvas is focused. - -### Workflow Actions -| Shortcut | Action | -|----------|--------| -| Mod+Enter | Run workflow (or cancel if running) | -| Mod+Z | Undo | -| Mod+Shift+Z | Redo | -| Mod+C | Copy selected blocks | -| Mod+X | Cut selected blocks | -| Mod+V | Paste blocks | -| Delete/Backspace | Delete selected blocks or edges | -| Shift+L | Auto-layout canvas | -| Mod+Shift+F | Fit to view | -| Mod+Shift+Enter | Accept Copilot changes | - -### Panel Navigation -| Shortcut | Action | -|----------|--------| -| Mod+F | Open workflow search and replace | -| Mod+Alt+F | Focus Toolbar search | - -### Global Navigation -| Shortcut | Action | -|----------|--------| -| Mod+K | Open search | -| Mod+Shift+A | Add new agent workflow | -| Mod+Shift+P | Create workflow | -| Mod+B | Toggle sidebar | -| Mod+L | Go to logs | - -### Utility -| Shortcut | Action | -|----------|--------| -| Mod+D | Clear terminal console | - -### Mouse Controls -| Action | Control | -|--------|---------| -| Pan/move canvas | Left-drag on empty space (hand mode, the default), middle-drag, scroll, or trackpad | -| Select multiple blocks | Shift+drag to draw a selection box. In cursor mode, left-drag on empty space draws it instead | -| Drag block | Left-drag on block header | -| Add to selection | Mod+Click or Shift+Click on blocks | - -## Quick Reference — Workspaces -| Action | How | -|--------|-----| -| Create workspace | Click workspace dropdown → New Workspace | -| Switch workspaces | Click workspace dropdown → Select workspace | -| Invite teammates | Sidebar → Invite | -| Rename/Duplicate/Export/Delete workspace | Right-click workspace → action | - -## Quick Reference — Workflows -| Action | How | -|--------|-----| -| Create workflow | Click + button in sidebar | -| Reorder/move workflows | Drag workflow up/down or onto a folder | -| Import workflow | Click import button in sidebar → Select file | -| Multi-select workflows | Mod+Click or Shift+Click workflows in sidebar | -| Open in new tab | Right-click workflow → Open in New Tab | -| Rename/Duplicate/Export/Delete | Right-click workflow → action | - -## Quick Reference — Blocks -| Action | How | -|--------|-----| -| Add a block | Drag from Toolbar panel, or right-click canvas → Add Block | -| Multi-select blocks | Mod+Click or Shift+Click additional blocks, or Shift+drag a selection box | -| Copy/Paste blocks | Mod+C / Mod+V | -| Duplicate/Delete blocks | Right-click → action | -| Rename a block | Click block name in header | -| Enable/Disable block | Right-click → Enable/Disable | -| Lock/Unlock block | Hover block → Click lock icon (Admin only) | -| Toggle handle orientation | Right-click → Toggle Handles | -| Open a block in the Editor panel | Right-click → Open Editor | -| Move a block out of a loop/parallel | Right-click → Remove from Subflow | -| Configure a block | Select block → use Editor panel on right | - -## Quick Reference — Connections -| Action | How | -|--------|-----| -| Create connection | Drag from output handle to input handle | -| Delete connection | Click edge to select → Delete key | -| Use output in another block | Drag connection tag into input field | - -## Quick Reference — Running & Testing -| Action | How | -|--------|-----| -| Run workflow | Click Run Workflow button or Mod+Enter | -| Stop workflow | Click Stop button or Mod+Enter while running | -| Test with chat | Use Chat panel on the right side | -| Run from block | Hover block → Click play button, or right-click → Run from block | -| Run until block | Right-click block → Run until block | -| View execution logs | Open terminal panel at bottom, or Mod+L | -| Filter/Search/Copy/Clear logs | Terminal panel controls | - -## Quick Reference — Deployment -| Action | How | -|--------|-----| -| Deploy workflow | Click Deploy button in panel | -| Update deployment | Click Update when changes are detected | -| Revert deployment | Previous versions in Deploy tab → Promote to live | -| Copy API endpoint | Deploy tab → API → Copy API cURL | - -## Quick Reference — Variables -| Action | How | -|--------|-----| -| Add/Edit/Delete workflow variable | Panel → Variables → Add Variable | -| Add environment variable | Settings → Environment Variables → Add | -| Reference workflow variable | Use syntax | -| Reference environment variable | Use {{ENV_VAR}} syntax | -` diff --git a/apps/sim/lib/copilot/tools/handlers/platform.ts b/apps/sim/lib/copilot/tools/handlers/platform.ts deleted file mode 100644 index f5cc43f910b..00000000000 --- a/apps/sim/lib/copilot/tools/handlers/platform.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' -import { PLATFORM_ACTIONS_CONTENT } from './platform-actions' - -export async function executeGetPlatformActions( - _rawParams: Record, - _context: ExecutionContext -): Promise { - return { success: true, output: { content: PLATFORM_ACTIONS_CONTENT } } -} diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.test.ts b/apps/sim/lib/copilot/tools/handlers/vfs.test.ts index 72eea0cefb9..0072f1f8d90 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants' const { getOrMaterializeVFS } = vi.hoisted(() => ({ @@ -405,3 +405,109 @@ describe('vfs uploads are opt-in (like recently-deleted/)', () => { expect(grepChatUpload).toHaveBeenCalledWith('report.json', 'chat-1', 'x', expect.any(Object)) }) }) + +describe('vfs handlers docs corpus routing', () => { + const fetchMock = vi.fn() + const DOCS_PAGE = 'docs/workflows/blocks/agent.mdx' + + beforeEach(() => { + vi.clearAllMocks() + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('globs the docs corpus without materializing the workspace VFS', async () => { + const result = await executeVfsGlob({ pattern: 'docs/**' }, GREP_CTX) + + expect(result.success).toBe(true) + expect((result.output as { files: string[] }).files).toContain(DOCS_PAGE) + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('reads a docs page via the live-site fetch, not the workspace VFS', async () => { + fetchMock.mockResolvedValue({ ok: true, status: 200, text: async () => 'line one\nline two' }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(true) + expect(result.output).toEqual({ content: 'line one\nline two', totalLines: 2 }) + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('surfaces DocsCorpusError messages verbatim from read, without fetching', async () => { + const unknown = await executeVfsRead({ path: 'docs/not-a-real-page.mdx' }, GREP_CTX) + expect(unknown.success).toBe(false) + expect(unknown.error).toContain('Docs page not found') + + const dir = await executeVfsRead({ path: 'docs/workflows/blocks' }, GREP_CTX) + expect(dir.success).toBe(false) + expect(dir.error).toContain('is a directory') + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('greps exactly one docs page and rejects multi-page scopes verbatim', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => 'alpha\ncron beta\ngamma', + }) + + const single = await executeVfsGrep({ pattern: 'cron', path: DOCS_PAGE }, GREP_CTX) + expect(single.success).toBe(true) + + const multi = await executeVfsGrep({ pattern: 'cron', path: 'docs/workflows' }, GREP_CTX) + expect(multi.success).toBe(false) + expect(multi.error).toContain('single page') + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('truncates an oversized multi-line docs page to fit the inline cap', async () => { + const line = 'y'.repeat(200) + const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1)) + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => Array.from({ length: totalLines }, () => line).join('\n'), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(true) + const output = result.output as { content: string; totalLines: number } + expect(output.totalLines).toBe(totalLines) + expect(output.content).toContain('[Page truncated: returned lines 1-') + expect(JSON.stringify(output).length).toBeLessThanOrEqual(TOOL_RESULT_MAX_INLINE_CHARS) + }) + + it('fails a docs page whose single line cannot fit inline instead of returning it oversized', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => 'z'.repeat(TOOL_RESULT_MAX_INLINE_CHARS + 1000), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(false) + expect(result.error).toContain('Grep this page') + }) + + it('rejects an explicit window that still overflows instead of truncating it', async () => { + const line = 'y'.repeat(200) + const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1)) + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => Array.from({ length: totalLines }, () => line).join('\n'), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE, offset: 0, limit: totalLines }, GREP_CTX) + + expect(result.success).toBe(false) + expect(result.error).toContain('still too large over the requested window') + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.ts b/apps/sim/lib/copilot/tools/handlers/vfs.ts index 0a1992ddd3c..cb270517f48 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.ts @@ -2,6 +2,14 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants' +import { + couldMatchDocsScope, + DocsCorpusError, + globDocs, + grepDocsPage, + isDocsPath, + readDocsPage, +} from '@/lib/copilot/docs/docs-corpus' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' import { getOrMaterializeVFS } from '@/lib/copilot/vfs' import type { GrepCountEntry, GrepMatch } from '@/lib/copilot/vfs/operations' @@ -79,6 +87,38 @@ function hasModelAttachment(result: unknown): boolean { ) } +/** + * Trim an oversized docs page to the largest whole-line prefix that fits the + * inline budget, preserving the true `totalLines` so the model can page through + * the rest with offset/limit. Returns null when not even one line fits — a + * single line longer than the cap — so the caller can fail instead of returning + * an over-cap payload as success. + */ +function truncateDocsPageToInlineCap(page: { content: string; totalLines: number }): { + output: { content: string; totalLines: number } + returnedLines: number +} | null { + const lines = page.content.split('\n') + // Route to ONE more fetch, not two. Telling the model to grep and then read + // costs two more uncached fetches of a page it already partly has; grep and + // read cost the same single fetch, so grep is an alternative to a read here, + // never a step before one. + const notice = (shown: number) => + `\n\n[Page truncated: returned lines 1-${shown} of ${page.totalLines}. To continue, read this path with offset: ${shown}. To jump straight to a section, grep this path INSTEAD of reading it — grep is the same single fetch and returns only matching lines with their numbers.]` + + let kept = lines.length + while (kept > 0) { + const content = `${lines.slice(0, kept).join('\n')}${notice(kept)}` + if ( + serializedResultSize({ content, totalLines: page.totalLines }) <= TOOL_RESULT_MAX_INLINE_CHARS + ) { + return { output: { content, totalLines: page.totalLines }, returnedLines: kept } + } + kept = Math.floor(kept / 2) + } + return null +} + export async function executeVfsGrep( params: Record, context: ExecutionContext @@ -107,13 +147,16 @@ export async function executeVfsGrep( // Routing mirrors read/glob: // - uploads/ -> grep one chat upload's content (chat-scoped) + // - docs/ -> grep one docs.sim.ai page (one page only — each is a fetch) // - files/ -> grep one workspace file's content (one file only) // - everything else -> grep the in-memory VFS map (workflow JSON, metadata) - // Chat uploads are opt-in like recently-deleted/: they are never in the VFS - // map, so an unscoped grep can't touch them — only an explicit uploads/ - // path does, and only one upload at a time. + // Chat uploads and the docs corpus are opt-in like recently-deleted/: they are + // never in the VFS map, so an unscoped grep can't touch them — only an explicit + // uploads/ or docs/ path does, and only one at a time. let result: GrepMatch[] | string[] | GrepCountEntry[] - if (isChatUploadGrepPath(rawPath)) { + if (rawPath !== undefined && isDocsPath(rawPath)) { + result = await grepDocsPage(rawPath, pattern, grepOptions) + } else if (isChatUploadGrepPath(rawPath)) { if (!context.chatId) { return { success: false, error: 'No chat context available for uploads/' } } @@ -157,8 +200,8 @@ export async function executeVfsGrep( } catch (err) { // Expected single-file scoping / no-text / too-large conditions: surface the // message verbatim instead of logging an internal failure. - if (err instanceof WorkspaceFileGrepError) { - logger.debug('vfs_grep workspace file rejected', { + if (err instanceof WorkspaceFileGrepError || err instanceof DocsCorpusError) { + logger.debug('vfs_grep single-file scope rejected', { pattern, path: rawPath, error: err.message, @@ -189,6 +232,15 @@ export async function executeVfsGlob( } try { + // The docs corpus is a lazy view of docs.sim.ai built from the generated + // manifest, not part of the workspace VFS — an explicit docs/ pattern is the + // only way to see it. + if (couldMatchDocsScope(pattern)) { + const files = globDocs(pattern) + logger.debug('vfs_glob docs result', { pattern, fileCount: files.length }) + return { success: true, output: { files } } + } + const vfs = await getGatedVFS(workspaceId, context.userId) let files = vfs.glob(pattern) @@ -248,6 +300,41 @@ export async function executeVfsRead( } } + // Docs pages are fetched from the live docs site on demand — the manifest + // path is the URL path, so there is nothing workspace-scoped to resolve. + if (isDocsPath(path)) { + const page = await readDocsPage(path) + const windowed = applyWindow(page) + if (serializedResultSize(windowed) > TOOL_RESULT_MAX_INLINE_CHARS) { + // Several real docs pages (the largest integration references) exceed the + // inline cap, so failing here would make a plain read of them always fail + // and cost a second fetch to recover. Truncate to what fits instead and + // tell the model how to page — but only when it did not ask for a window, + // since an explicit offset/limit that still overflows is a caller error. + if (offset !== undefined || limit !== undefined) { + return { + success: false, + error: `${path} is still too large over the requested window. Narrow offset/limit, or grep this page for the section you need.`, + } + } + const truncated = truncateDocsPageToInlineCap(page) + if (!truncated) { + return { + success: false, + error: `${path} is too large to return inline even truncated. Grep this page for the section you need.`, + } + } + logger.debug('vfs_read truncated oversized docs page', { + path, + totalLines: page.totalLines, + returnedLines: truncated.returnedLines, + }) + return { success: true, output: truncated.output } + } + logger.debug('vfs_read resolved docs page', { path, totalLines: page.totalLines }) + return { success: true, output: windowed } + } + // Handle chat-scoped uploads via the uploads/ virtual prefix. // Uploads are flat and have no metadata/content split like files/ — the upload // IS the first path segment after uploads/. Any trailing segment (e.g. a @@ -367,6 +454,12 @@ export async function executeVfsRead( output: result, } } catch (err) { + // Expected docs-corpus conditions (unknown page, directory path, site + // unreachable): surface the message verbatim. + if (err instanceof DocsCorpusError) { + logger.debug('vfs_read docs page rejected', { path, error: err.message }) + return { success: false, error: err.message } + } logger.error('vfs_read failed', { path, error: toError(err).message, diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts new file mode 100644 index 00000000000..c3f406c56b9 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts @@ -0,0 +1,45 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1' +import { isKnownTool, isSimExecuted } from '@/lib/copilot/tool-executor/router' +import { getHiddenToolNames } from '@/lib/copilot/tools/client/hidden-tools' +import { getRegisteredServerToolNames } from '@/lib/copilot/tools/server/router' + +/** + * `executeTool` gates on `isKnownTool` (catalog membership) before it ever + * consults the handler registry, so a sim-routed tool needs every link of this + * chain or dispatch rejects it before the handler is reached. These assertions + * pin that chain for search_docs. + */ +describe('search_docs dispatch chain', () => { + it('is in the catalog, so dispatch does not reject it as unknown', () => { + expect(isKnownTool('search_docs')).toBe(true) + }) + + it('routes to sim, so dispatch reaches the server tool registry', () => { + expect(isSimExecuted('search_docs')).toBe(true) + }) + + it('has a registered server handler', () => { + expect(getRegisteredServerToolNames()).toContain('search_docs') + }) +}) + +/** + * The retired ids are fully unregistered server-side — no catalog entry, no + * handler, no alias. Only the client-side chip suppression survives, forever, + * so historical persisted chats replay without rendering chips for tools that + * no longer exist (the load_agent_skill precedent). + */ +describe('retired docs-tool ids', () => { + for (const retired of ['search_documentation', 'get_platform_actions']) { + it(`${retired} is gone from the catalog and server registry but stays chip-hidden`, () => { + expect(TOOL_CATALOG[retired]).toBeUndefined() + expect(isKnownTool(retired)).toBe(false) + expect(getRegisteredServerToolNames()).not.toContain(retired) + expect(getHiddenToolNames().has(retired)).toBe(true) + }) + } +}) diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts new file mode 100644 index 00000000000..37318f5fc53 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts @@ -0,0 +1,102 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { DocsSearchOutcome } from '@/lib/copilot/docs/docs-search' + +const { mockSearchDocs } = vi.hoisted(() => ({ + mockSearchDocs: vi.fn(), +})) + +vi.mock('@/lib/copilot/docs/docs-search', () => ({ + searchDocs: mockSearchDocs, +})) + +import { searchDocsServerTool } from '@/lib/copilot/tools/server/docs/search-docs' + +function outcome(overrides: Partial): DocsSearchOutcome { + return { + results: [], + candidatesConsidered: 0, + droppedBelowThreshold: 0, + droppedStale: 0, + ...overrides, + } +} + +const RESULT = { + path: 'docs/agents.mdx', + url: 'https://docs.sim.ai/agents', + title: 'Agents', + content: 'body', + similarity: 0.9, +} + +describe('searchDocsServerTool', () => { + beforeEach(() => { + mockSearchDocs.mockReset() + }) + + it('forwards query, path, and topK to the search layer', async () => { + mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 })) + + const output = await searchDocsServerTool.execute({ + query: 'how do agents work', + path: 'docs/agents.mdx', + topK: 7, + }) + + expect(mockSearchDocs).toHaveBeenCalledWith('how do agents work', { + path: 'docs/agents.mdx', + topK: 7, + }) + expect(output).toEqual({ + results: [RESULT], + query: 'how do agents work', + totalResults: 1, + }) + }) + + it('omits the note when nothing was dropped', async () => { + mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 })) + + const output = await searchDocsServerTool.execute({ query: 'q' }) + + expect(output.note).toBeUndefined() + }) + + it('explains an empty result set caused by filtering, so it does not read as missing docs', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ candidatesConsidered: 2, droppedBelowThreshold: 1, droppedStale: 1 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }) + + expect(output.note).toContain('does NOT mean the docs lack this topic') + expect(output.note).toContain('1 scored too low') + expect(output.note).toContain('1 point at pages no longer in the docs') + }) + + it('notes threshold-only drops on a partial result set', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ results: [RESULT], candidatesConsidered: 3, droppedBelowThreshold: 2 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }) + + expect(output.note).toContain('Returned 1 of 3 candidate(s)') + expect(output.note).toContain('2 scored too low') + expect(output.note).not.toContain('no longer in the docs') + }) + + it('notes stale-only drops on a partial result set', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ results: [RESULT], candidatesConsidered: 2, droppedStale: 1 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }) + + expect(output.note).toContain('1 point at pages no longer in the docs') + expect(output.note).not.toContain('scored too low') + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs.ts new file mode 100644 index 00000000000..96bdc922e2c --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs.ts @@ -0,0 +1,65 @@ +import type { DocsSearchResult } from '@/lib/copilot/docs/docs-search' +import { searchDocs } from '@/lib/copilot/docs/docs-search' +import { SearchDocs } from '@/lib/copilot/generated/tool-catalog-v1' +import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' + +interface SearchDocsParams { + query: string + topK?: number + path?: string +} + +interface SearchDocsOutput { + results: DocsSearchResult[] + query: string + totalResults: number + /** + * Present only when the vector search matched chunks that were then filtered + * out. Without it an empty result set reads as "the docs do not cover this", + * which sends the caller off to guess instead of rephrasing or falling back + * to glob. + */ + note?: string +} + +/** + * Explain a short or empty result set in terms the caller can act on. Returns + * undefined when nothing was dropped — the common case needs no commentary. + */ +function shortfallNote(outcome: Awaited>): string | undefined { + const { results, candidatesConsidered, droppedBelowThreshold, droppedStale } = outcome + if (droppedBelowThreshold === 0 && droppedStale === 0) return undefined + + const reasons: string[] = [] + if (droppedBelowThreshold > 0) + reasons.push(`${droppedBelowThreshold} scored too low to be relevant`) + if (droppedStale > 0) { + reasons.push( + `${droppedStale} point at pages no longer in the docs (the search index lags the site)` + ) + } + const dropped = reasons.join(' and ') + + return results.length === 0 + ? `No relevant matches. The search index returned ${candidatesConsidered} candidate(s), but ${dropped} — this does NOT mean the docs lack this topic. Rephrase the query, widen it by dropping the path scope, or browse with glob("docs/**").` + : `Returned ${results.length} of ${candidatesConsidered} candidate(s); ${dropped}. Rephrase or widen the query if these look off-topic.` +} + +/** + * Vector search over Sim's product documentation, scoped to the same pages the + * agent can `read` from the `docs/` VFS tree. Search-agent only; the corpus + * logic lives in `@/lib/copilot/docs/docs-search`. + */ +export const searchDocsServerTool: BaseServerTool = { + name: SearchDocs.id, + async execute(params: SearchDocsParams): Promise { + const outcome = await searchDocs(params.query, { path: params.path, topK: params.topK }) + const note = shortfallNote(outcome) + return { + results: outcome.results, + query: params.query, + totalResults: outcome.results.length, + ...(note ? { note } : {}), + } + }, +} diff --git a/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts b/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts deleted file mode 100644 index db8f9e73da8..00000000000 --- a/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { db } from '@sim/db' -import { docsEmbeddings } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { sql } from 'drizzle-orm' -import { SearchDocumentation } from '@/lib/copilot/generated/tool-catalog-v1' -import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' -import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' - -interface DocsSearchParams { - query: string - topK?: number - threshold?: number -} - -const DEFAULT_DOCS_SIMILARITY_THRESHOLD = 0.3 - -export const searchDocumentationServerTool: BaseServerTool = { - name: SearchDocumentation.id, - async execute(params: DocsSearchParams): Promise { - const logger = createLogger('SearchDocumentationServerTool') - const { query, topK = 10, threshold } = params - if (!query || typeof query !== 'string') throw new Error('query is required') - - logger.info('Executing docs search', { query, topK }) - - const similarityThreshold = threshold ?? DEFAULT_DOCS_SIMILARITY_THRESHOLD - - const { embedding: queryEmbedding } = await generateSearchEmbedding(query) - if (!queryEmbedding || queryEmbedding.length === 0) { - return { results: [], query, totalResults: 0 } - } - - const results = await db - .select({ - chunkId: docsEmbeddings.chunkId, - chunkText: docsEmbeddings.chunkText, - sourceDocument: docsEmbeddings.sourceDocument, - sourceLink: docsEmbeddings.sourceLink, - headerText: docsEmbeddings.headerText, - headerLevel: docsEmbeddings.headerLevel, - similarity: sql`1 - (${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector)`, - }) - .from(docsEmbeddings) - .orderBy(sql`${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector`) - .limit(topK) - - const filteredResults = results.filter((r) => r.similarity >= similarityThreshold) - const documentationResults = filteredResults.map((r, idx) => ({ - id: idx + 1, - title: String(r.headerText || 'Untitled Section'), - url: String(r.sourceLink || '#'), - content: String(r.chunkText || ''), - similarity: r.similarity, - })) - - logger.info('Docs search complete', { count: documentationResults.length }) - return { results: documentationResults, query, totalResults: documentationResults.length } - }, -} diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index 266c967bf57..e536ae3b39f 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -23,7 +23,7 @@ import { } from '@/lib/copilot/tools/server/base-tool' import { getBlocksMetadataServerTool } from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool' import { getTriggerBlocksServerTool } from '@/lib/copilot/tools/server/blocks/get-trigger-blocks' -import { searchDocumentationServerTool } from '@/lib/copilot/tools/server/docs/search-documentation' +import { searchDocsServerTool } from '@/lib/copilot/tools/server/docs/search-docs' import { enrichmentRunServerTool } from '@/lib/copilot/tools/server/enrichment/enrichment-run' import { createFileServerTool } from '@/lib/copilot/tools/server/files/create-file' import { downloadToWorkspaceFileServerTool } from '@/lib/copilot/tools/server/files/download-to-workspace-file' @@ -158,7 +158,7 @@ const baseServerToolRegistry: Record = { [editWorkflowServerTool.name]: editWorkflowServerTool, [queryLogsServerTool.name]: queryLogsServerTool, [getJobLogsServerTool.name]: getJobLogsServerTool, - [searchDocumentationServerTool.name]: searchDocumentationServerTool, + [searchDocsServerTool.name]: searchDocsServerTool, [searchOnlineServerTool.name]: searchOnlineServerTool, [setEnvironmentVariablesServerTool.name]: setEnvironmentVariablesServerTool, [getCredentialsServerTool.name]: getCredentialsServerTool, diff --git a/apps/sim/lib/copilot/tools/tool-display.test.ts b/apps/sim/lib/copilot/tools/tool-display.test.ts index feaa674f753..bbd9d6c5b1a 100644 --- a/apps/sim/lib/copilot/tools/tool-display.test.ts +++ b/apps/sim/lib/copilot/tools/tool-display.test.ts @@ -78,6 +78,26 @@ describe('getToolDisplayTitle natural-language coverage', () => { expect(getToolDisplayTitle('diff_workflows')).toBe('Comparing workflows') }) + it('includes the query in search_docs titles', () => { + expect(getToolDisplayTitle('search_docs')).toBe('Searching Sim docs') + expect(getToolDisplayTitle('search_docs', { query: 'loop blocks iteration' })).toBe( + 'Searching Sim docs for "loop blocks iteration"' + ) + // The completed-state flip must keep the suffix, not drop back to the bare label. + expect( + getToolCompletedTitle( + getToolDisplayTitle('search_docs', { query: 'how to read workflow logs' }) + ) + ).toBe('Searched Sim docs for "how to read workflow logs"') + // A long agent-written query is truncated rather than blowing out the chip. + expect( + getToolDisplayTitle('search_docs', { + query: + 'reference block outputs connection tags blockname.field pass data between blocks in a workflow', + })?.length + ).toBeLessThanOrEqual('Searching Sim docs for ""'.length + 60 + '...'.length) + }) + it('falls back to running code for function_execute without a title', () => { expect(getToolDisplayTitle('function_execute')).toBe('Running code') expect(getToolDisplayTitle('function_execute', { title: 'Crunching numbers' })).toBe( diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index 091d289a5b1..5053e1c0ad2 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -1,4 +1,4 @@ -import { stripVersionSuffix } from '@sim/utils/string' +import { stripVersionSuffix, truncate } from '@sim/utils/string' /** * Single source of truth for copilot tool-call display titles. @@ -451,7 +451,6 @@ const TOOL_TITLES: Record = { get_block_upstream_references: 'Getting block references', get_deployed_workflow_state: 'Getting deployed workflow', get_deployment_log: 'Getting deployment logs', - get_platform_actions: 'Getting platform actions', get_scheduled_task_logs: 'Getting scheduled task logs', get_workflow_data: 'Getting workflow data', get_workflow_run_options: 'Getting run options', @@ -473,7 +472,7 @@ const TOOL_TITLES: Record = { rename_workflow: 'Renaming workflow', restore_resource: 'Restoring resource', run_block: 'Running block', - search_documentation: 'Searching documentation', + search_docs: 'Searching Sim docs', search_patterns: 'Searching patterns', set_block_enabled: 'Toggling block', set_environment_variables: 'Setting environment variables', @@ -765,6 +764,10 @@ export function getToolDisplayTitle(name: string, args?: Record const target = firstStringArg(args, 'toolTitle', 'title') return target ? `Searching online for ${target}` : 'Searching online' } + case 'search_docs': { + const target = firstStringArg(args, 'toolTitle', 'title', 'query') + return target ? `Searching Sim docs for "${truncate(target, 60)}"` : 'Searching Sim docs' + } case 'grep': { const target = firstStringArg(args, 'toolTitle', 'title') return target ? `Searching for ${target}` : 'Searching' diff --git a/apps/sim/stores/panel/types.ts b/apps/sim/stores/panel/types.ts index dbf91a502a4..758ae746986 100644 --- a/apps/sim/stores/panel/types.ts +++ b/apps/sim/stores/panel/types.ts @@ -28,6 +28,11 @@ export type ChatContext = | { kind: 'folder'; folderId: string; label: string } | { kind: 'filefolder'; fileFolderId: string; label: string } | { kind: 'scheduledtask'; scheduleId: string; label: string } + /** + * Inert today: the server drops it in processContextsServer. Kept so a + * future @Docs mention can be revived on the search_docs backend instead of + * reintroducing the kind across the wire schema and this union. + */ | { kind: 'docs'; label: string } /** * A tab in the desktop browser or terminal panel, dragged into the input to diff --git a/package.json b/package.json index 3367e07051f..6e153ba6dd8 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,8 @@ "metrics-contract:check": "bun run scripts/sync-metrics-contract.ts --check", "vfs-snapshot-contract:generate": "bun run scripts/sync-vfs-snapshot-contract.ts", "vfs-snapshot-contract:check": "bun run scripts/sync-vfs-snapshot-contract.ts --check", + "docs-manifest:generate": "bun run scripts/sync-docs-manifest.ts", + "docs-manifest:check": "bun run scripts/sync-docs-manifest.ts --check", "mship:generate": "bun run scripts/generate-mship-contracts.ts", "mship:check": "bun run scripts/generate-mship-contracts.ts --check", "library:covers": "bun run scripts/generate-library-covers.tsx", diff --git a/scripts/sync-docs-manifest.ts b/scripts/sync-docs-manifest.ts new file mode 100644 index 00000000000..7373a72e26c --- /dev/null +++ b/scripts/sync-docs-manifest.ts @@ -0,0 +1,114 @@ +/** + * Generate the static docs manifest the copilot's `docs/` VFS tree is built from. + * + * Source of truth: `apps/docs/content/docs/en/**\/*.mdx` — the English docs + * corpus, whose folder structure mirrors the public docs.sim.ai URL structure. + * The copilot never reads those files from disk (they are not deployed with + * `apps/sim`); it globs this manifest for structure and fetches page content + * from the live site on demand. That makes the manifest the one thing that can + * drift, hence `--check` in CI. + * + * Path derivation (each entry is BOTH the `docs/`-relative VFS path and the + * docs.sim.ai URL path, so a read is a plain fetch of `https://docs.sim.ai/`): + * - `workflows/blocks/agent.mdx` → `workflows/blocks/agent.mdx` + * - `workflows/index.mdx` → `workflows.mdx` (fumadocs folds index pages + * into their parent URL; `/workflows/index.mdx` + * is a 404 on the site) + * + * Excluded, and intentionally absent from the VFS: every section in + * `UNMOUNTED_DOCS_SECTIONS` (fetch those with the scrape tool if ever needed), + * the root `index.mdx` (its URL is `/`, which redirects), and every non-`en` + * locale. + * + * Usage: + * bun run docs-manifest:generate # write the manifest + * bun run docs-manifest:check # fail (exit 1) if the manifest is stale + */ +import { readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { foldDocsIndexPath, UNMOUNTED_DOCS_SECTIONS } from '../apps/sim/lib/copilot/docs/docs-path' +import { formatGeneratedSource } from './format-generated-source' + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) +const ROOT = resolve(SCRIPT_DIR, '..') +const DOCS_CONTENT_DIR = resolve(ROOT, 'apps/docs/content/docs/en') +const OUTPUT_PATH = resolve(ROOT, 'apps/sim/lib/copilot/generated/docs-manifest.ts') + +/** + * Top-level docs sections deliberately left out of the copilot's `docs/` tree. + * Shared with the vector search's unscoped filter so readability and + * findability cannot drift apart — see `UNMOUNTED_DOCS_SECTIONS`. + */ +const EXCLUDED_SECTIONS = new Set(UNMOUNTED_DOCS_SECTIONS) + +/** Collect every `.mdx` file under `dir`, as paths relative to {@link DOCS_CONTENT_DIR}. */ +async function collectMdxPaths(dir: string, prefix = ''): Promise { + const entries = await readdir(dir, { withFileTypes: true }) + const paths: string[] = [] + for (const entry of entries) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name + if (entry.isDirectory()) { + if (prefix === '' && EXCLUDED_SECTIONS.has(entry.name)) continue + paths.push(...(await collectMdxPaths(resolve(dir, entry.name), relative))) + continue + } + if (entry.isFile() && entry.name.endsWith('.mdx')) paths.push(relative) + } + return paths +} + +/** Map an `en`-relative mdx file path to its docs.sim.ai URL path, or null to drop it. */ +function toDocsPath(mdxPath: string): string | null { + if (mdxPath === 'index.mdx') return null + return foldDocsIndexPath(mdxPath) +} + +function render(paths: string[]): string { + const entries = paths.map((path) => ` '${path}',`).join('\n') + return `// AUTO-GENERATED FILE. DO NOT EDIT. +// Generated from apps/docs/content/docs/en by scripts/sync-docs-manifest.ts +// Run: bun run docs-manifest:generate +// + +/** + * Every page in the copilot's read-only \`docs/\` VFS tree, as a path that is + * simultaneously the \`docs/\`-relative VFS path and the docs.sim.ai URL path + * (so \`docs/workflows/blocks/agent.mdx\` reads + * \`https://docs.sim.ai/workflows/blocks/agent.mdx\`). Sorted. + */ +export const DOCS_MANIFEST: readonly string[] = [ +${entries} +] +` +} + +async function main() { + const checkOnly = process.argv.includes('--check') + + const mdxPaths = await collectMdxPaths(DOCS_CONTENT_DIR) + const docsPaths = mdxPaths + .map(toDocsPath) + .filter((path): path is string => path !== null) + .sort() + + if (docsPaths.length === 0) { + throw new Error(`No docs pages found under ${DOCS_CONTENT_DIR}`) + } + + const rendered = formatGeneratedSource(render(docsPaths), OUTPUT_PATH, ROOT) + + if (checkOnly) { + const existing = await readFile(OUTPUT_PATH, 'utf8').catch(() => null) + if (existing !== rendered) { + throw new Error( + 'Generated docs manifest is stale — the docs tree changed (page added, removed, or renamed). Run: bun run docs-manifest:generate' + ) + } + return + } + + await writeFile(OUTPUT_PATH, rendered, 'utf8') +} + +await main()