Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions src/lib/mcp/analytics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
clientCapabilityAnalyticsFromInitialize,
enrichMcpAnalyticsEvent,
instrumentMcpAnalytics,
intentFromContextArgument,
MCP_CAPABILITY_REQUESTED_EVENT,
MCP_CLIENT_ELICITATION_MODE_PROPERTY,
MCP_CLIENT_SUPPORTS_APPS_PROPERTY,
Expand Down Expand Up @@ -313,6 +314,41 @@ function toolCallEvent(properties: Record<string, unknown> = {}): CaptureEvent {
};
}

describe("intentFromContextArgument", () => {
test("returns the trimmed context argument", () => {
expect(
intentFromContextArgument({
params: {
name: "manage_browsers",
arguments: { action: "list", context: " Listing live browsers. " },
},
}),
).toBe("Listing live browsers.");
});

test("returns undefined when context is missing, blank, or not a string", () => {
expect(intentFromContextArgument({})).toBeUndefined();
expect(intentFromContextArgument({ params: {} })).toBeUndefined();
expect(
intentFromContextArgument({ params: { arguments: { action: "list" } } }),
).toBeUndefined();
expect(
intentFromContextArgument({ params: { arguments: { context: "" } } }),
).toBeUndefined();
expect(
intentFromContextArgument({ params: { arguments: { context: " \n" } } }),
).toBeUndefined();
expect(
intentFromContextArgument({ params: { arguments: { context: 42 } } }),
).toBeUndefined();
expect(
intentFromContextArgument({
params: { arguments: { context: { summary: "object" } } },
}),
).toBeUndefined();
});
});

describe("sanitizeMcpAnalyticsEvent", () => {
test("keeps $groups so every event carries org attribution", async () => {
const event = toolCallEvent({ $groups: { organization: "org_123" } });
Expand Down Expand Up @@ -1189,6 +1225,12 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
server.registerTool("ping", { inputSchema: z.object({}) }, async () => ({
content: [{ type: "text" as const, text: "pong" }],
}));
// Declares context itself, like manage_browsers, so the SDK does not own the argument.
server.registerTool(
"ping_with_context",
{ inputSchema: z.object({ context: z.string().optional() }) },
async () => ({ content: [{ type: "text" as const, text: "pong" }] }),
);
return server;
}

Expand Down Expand Up @@ -1303,6 +1345,28 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
expect(toolCall.properties.$session_id).toBe("ses_integration");
});

test("records intent for a tool that declares its own context argument", async () => {
const captured: { event?: string }[] = [];

const result = (await simulateRequest(captured, "tools/call", {
name: "ping_with_context",
arguments: {
context: "Checking intent capture on a tool-owned context.",
},
})) as { isError?: boolean };

expect(result.isError).not.toBe(true);
const toolCall = captured.find(
({ event }) => event === PostHogMCPAnalyticsEvent.ToolCall,
) as { properties: Record<string, unknown> };
expect(toolCall.properties).toMatchObject({
[PostHogMCPAnalyticsProperty.ToolName]: "ping_with_context",
[PostHogMCPAnalyticsProperty.Intent]:
"Checking intent capture on a tool-owned context.",
[PostHogMCPAnalyticsProperty.IntentSource]: "inferred",
});
});

test("classifies rejected capability input through instrumentation", async () => {
const captured: { event?: string }[] = [];

Expand Down
26 changes: 13 additions & 13 deletions src/lib/mcp/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import {
} from "@/lib/mcp/deprecated-params";
import {
type KernelFeedback,
KERNEL_FEEDBACK_TOOL_NAME,
LEGACY_SITE_COMPATIBILITY_FEEDBACK_TYPE,
legacyChallengeType,
registerFeedbackTool,
Expand Down Expand Up @@ -388,6 +387,17 @@ function sanitizeIntent(intent: string) {

const ANALYTICS_CONTEXT_PROPERTY = "__mcp_connection_analytics_context";

// PostHog reads the `context` argument only for tools whose schema leaves it to the SDK.
// Declaring `context` on a tool (the analytics-backed tools, manage_browsers) stops that
// read, so the SDK's fallback restores intent capture from the same argument.
export function intentFromContextArgument(request: {
params?: { arguments?: Record<string, unknown>; [key: string]: unknown };
}) {
const context = request.params?.arguments?.context;
const intent = typeof context === "string" ? context.trim() : "";
return intent || undefined;
}

/**
* Enforces the SENT_PROPERTIES allow-list on every event the SDK builds and sanitizes
* the free-form intent.
Expand Down Expand Up @@ -952,18 +962,8 @@ export function instrumentMcpAnalytics(
// the SDK default: it repeats per tool in every tools/list response, so it stays
// short, and it names the arguments agents must not copy into it.
context: { description: MCP_INTENT_ARGUMENT_DESCRIPTION },
// These tools declare their own context field, so PostHog no longer owns it.
intentFallback: (request) => {
if (
request.params?.name !== KERNEL_FEEDBACK_TOOL_NAME &&
request.params?.name !== KERNEL_MISSING_CAPABILITY_TOOL_NAME
)
return null;
const args = request.params?.arguments;
return isRecord(args) && typeof args.context === "string"
? args.context
: null;
},
// Runs for tools that declare their own context field, which PostHog then doesn't own.
intentFallback: intentFromContextArgument,
// A failed tool call otherwise fans out into a second `$exception` event whose
// `$exception_list` is built from the text the tool returned.
enableExceptionAutocapture: false,
Expand Down
84 changes: 84 additions & 0 deletions src/lib/mcp/tools/browser-vaults.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,90 @@ describe("browser vault attachment", () => {
}
});

test.each([
{
arguments: { action: "create", vault: "checkout" },
message:
'unknown key "vault": did you mean "vaults" (array of { id } or { name })?',
},
{
arguments: { action: "create", vaults: [{ name: "checkout" }], vualt: 1 },
message: 'unknown key "vualt" in manage_browsers arguments',
},
{
arguments: { action: "update", session_id: "brr_123", profile: "p" },
message: 'unknown key "profile" in manage_browsers arguments',
},
])(
"rejects unknown keys instead of dropping them ($message)",
async ({ arguments: args, message }) => {
let calls = 0;
const fixture = await connectTestMcp(registerBrowserCapabilities, {
browsers: {
create: async () => {
calls++;
return { session_id: "brr_123" };
},
update: async () => {
calls++;
return { session_id: "brr_123" };
},
},
});
try {
const result = await fixture.client.callTool({
name: "manage_browsers",
arguments: args,
});
expect(result.isError).toBe(true);
expect((result.content as Array<{ text: string }>)[0].text).toContain(
message,
);
expect(calls).toBe(0);
} finally {
await fixture.close();
}
},
);

test.each([
{ action: "create", arguments: { vaults: [{ name: "checkout" }] } },
{ action: "update", arguments: { session_id: "brr_123", name: "renamed" } },
])(
"accepts the analytics context argument without forwarding it ($action)",
async ({ action, arguments: args }) => {
const bodies: unknown[] = [];
const fixture = await connectTestMcp(registerBrowserCapabilities, {
browsers: {
create: async (body: unknown) => {
bodies.push(body);
return { session_id: "brr_123" };
},
update: async (_id: string, body: unknown) => {
bodies.push(body);
return { session_id: "brr_123" };
},
},
});
try {
const result = await fixture.client.callTool({
name: "manage_browsers",
arguments: {
action,
...args,
context:
"creating a browser to automate a checkout flow for the user",
},
});
expect(result.isError).toBeUndefined();
expect(bodies).toHaveLength(1);
expect(bodies[0]).not.toHaveProperty("context");
} finally {
await fixture.close();
}
},
);

test("does not retry a failed vault-bound browser creation", async () => {
let calls = 0;
const sdk = new Kernel({
Expand Down
Loading
Loading