diff --git a/.github/workflows/test-mcp-examples.yaml b/.github/workflows/test-mcp-examples.yaml new file mode 100644 index 000000000..969adf4c6 --- /dev/null +++ b/.github/workflows/test-mcp-examples.yaml @@ -0,0 +1,53 @@ +name: Test MCP Examples + +on: + push: + branches: + - main + paths: + - 'modules/ai-agents/examples/**/*.yaml' + - 'modules/ai-agents/examples/test-mcp-examples.sh' + pull_request: + branches: + - main + paths: + - 'modules/ai-agents/examples/**/*.yaml' + - 'modules/ai-agents/examples/test-mcp-examples.sh' + +jobs: + test-all-examples: + name: Test All MCP Examples + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install npm dependencies + run: npm install + + - name: Install doc-tools dependencies + run: npx --no-install doc-tools install-test-dependencies + + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + + - name: Make test script executable + run: chmod +x modules/ai-agents/examples/test-mcp-examples.sh + + - name: Run MCP examples tests + run: | + cd modules/ai-agents/examples + ./test-mcp-examples.sh + + - name: Test Summary + if: always() + run: | + echo "::notice title=MCP Examples Testing::All Cloud MCP examples have been validated" diff --git a/.github/workflows/update-extensions.yml b/.github/workflows/update-extensions.yml index e4f5c6c3b..1544f1135 100644 --- a/.github/workflows/update-extensions.yml +++ b/.github/workflows/update-extensions.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '22' - run: npm install diff --git a/.gitignore b/.gitignore index b2c51358d..d4ff844c4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build/_redirects poc-docs/ docs/ .docusaurus +.env diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 935c4bbbd..04afe1da4 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -37,6 +37,7 @@ asciidoc: - '@redpanda-data/docs-extensions-and-macros/macros/config-ref' - '@redpanda-data/docs-extensions-and-macros/macros/helm-ref' - '@redpanda-data/docs-extensions-and-macros/asciidoc-extensions/add-line-numbers-highlights' + - '@redpanda-data/docs-extensions-and-macros/macros/badge' antora: extensions: - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info' diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 1915748c7..3cff3873c 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -80,6 +80,9 @@ *** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] *** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] +**** xref:ai-agents:mcp/remote/manage-servers.adoc[Manage Servers] +**** xref:ai-agents:mcp/remote/scale-resources.adoc[Scale Resources] +**** xref:ai-agents:mcp/remote/monitor-activity.adoc[Monitor Activity] *** xref:ai-agents:mcp/remote/pipeline-patterns.adoc[MCP Server Patterns] * xref:develop:connect/about.adoc[Redpanda Connect] diff --git a/modules/ai-agents/examples/redpanda_output_with_processors.yaml b/modules/ai-agents/examples/redpanda_output_with_processors.yaml new file mode 100644 index 000000000..d58997a60 --- /dev/null +++ b/modules/ai-agents/examples/redpanda_output_with_processors.yaml @@ -0,0 +1,19 @@ +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topic: "llm-responses" + tls: + enabled: true + sasl: + - mechanism: "${REDPANDA_SASL_MECHANISM}" + username: "${REDPANDA_SASL_USERNAME}" + password: "${REDPANDA_SASL_PASSWORD}" + +processors: + - openai_chat_completion: + api_key: "${secrets.OPENAI_API_KEY}" + model: "gpt-4" + prompt: ${! json("question") } + - mapping: | + root.question = this.question + root.answer = this.content + root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") diff --git a/modules/ai-agents/examples/test-mcp-examples.sh b/modules/ai-agents/examples/test-mcp-examples.sh new file mode 100755 index 000000000..595fcebfe --- /dev/null +++ b/modules/ai-agents/examples/test-mcp-examples.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# Automated testing script for Redpanda Connect MCP examples +# +# Usage: +# ./test-mcp-examples.sh + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Counters +TOTAL=0 +SKIPPED=0 +MCP_FAILS=0 + +echo "๐Ÿงช Redpanda Connect MCP Examples Test Suite" +echo "============================================" +echo "" + +# Run MCP server lint on the directory +echo "Running rpk connect mcp-server lint..." +LINT_OUTPUT=$(rpk connect mcp-server lint --skip-env-var-check --verbose 2>&1) || { + echo -e "${RED}โŒ Linting failed${NC}" + echo "" + echo "$LINT_OUTPUT" + exit 1 +} +echo -e "${GREEN}โœ… Linting passed${NC}" +echo "" + +# Function to validate MCP metadata +validate_mcp_metadata() { + local file=$1 + + echo -n " Validating MCP metadata... " + + # Determine which YAML parser to use + local use_yq=true + if ! command -v yq &> /dev/null; then + use_yq=false + if ! command -v python3 &> /dev/null; then + echo -e "${RED}FAILED${NC} (neither yq nor python3 available)" + MCP_FAILS=$((MCP_FAILS + 1)) + return 1 + fi + fi + + # Check if .meta.mcp exists + local mcp_exists + if $use_yq; then + mcp_exists=$(yq eval '.meta.mcp' "$file" 2>/dev/null) + else + mcp_exists=$(python3 -c " +import yaml +try: + with open('$file') as f: + doc = yaml.safe_load(f) + meta = doc.get('meta', {}) if doc else {} + mcp = meta.get('mcp') + print('null' if mcp is None else 'exists') +except: + print('null') +" 2>/dev/null) + fi + + if [[ "$mcp_exists" == "null" || -z "$mcp_exists" ]]; then + echo -e "${YELLOW}SKIPPED${NC} (no MCP metadata)" + SKIPPED=$((SKIPPED + 1)) + return 0 + fi + + # Read .meta.mcp.enabled + local enabled + if $use_yq; then + enabled=$(yq eval '.meta.mcp.enabled' "$file" 2>/dev/null) + else + enabled=$(python3 -c " +import yaml +try: + with open('$file') as f: + doc = yaml.safe_load(f) + enabled = doc.get('meta', {}).get('mcp', {}).get('enabled') + print('null' if enabled is None else str(enabled).lower()) +except: + print('null') +" 2>/dev/null) + fi + + if [[ "$enabled" != "true" ]]; then + echo -e "${YELLOW}WARNING${NC} (mcp.enabled not set to true)" + return 0 + fi + + # Read .meta.mcp.description + local description + if $use_yq; then + description=$(yq eval '.meta.mcp.description' "$file" 2>/dev/null) + else + description=$(python3 -c " +import yaml +try: + with open('$file') as f: + doc = yaml.safe_load(f) + desc = doc.get('meta', {}).get('mcp', {}).get('description') + print('null' if desc is None or desc == '' else str(desc)) +except: + print('null') +" 2>/dev/null) + fi + + if [[ "$description" == "null" || -z "$description" ]]; then + echo -e "${RED}FAILED${NC} (missing description)" + MCP_FAILS=$((MCP_FAILS + 1)) + return 1 + fi + + echo -e "${GREEN}PASSED${NC}" + return 0 +} + +# Validate MCP metadata for each file +for file in *.yaml; do + if [[ -f "$file" ]]; then + TOTAL=$((TOTAL + 1)) + echo "" + echo -e "${BLUE}๐Ÿ“„ Validating: $file${NC}" + validate_mcp_metadata "$file" + fi +done + +# Summary +echo "" +echo "============================================" +echo "๐Ÿ“Š Test Summary" +echo "============================================" +echo "Total configs tested: $TOTAL" +if [[ $MCP_FAILS -gt 0 ]]; then + echo -e "MCP validation failures: ${RED}$MCP_FAILS${NC}" +fi +if [[ $SKIPPED -gt 0 ]]; then + echo -e "Skipped: ${YELLOW}$SKIPPED${NC}" +fi +echo "" + +if [[ $MCP_FAILS -gt 0 ]]; then + echo -e "${RED}โŒ Some tests failed${NC}" + exit 1 +else + echo -e "${GREEN}โœ… All tests passed!${NC}" + exit 0 +fi diff --git a/modules/ai-agents/examples/testing.adoc b/modules/ai-agents/examples/testing.adoc new file mode 100644 index 000000000..812400b24 --- /dev/null +++ b/modules/ai-agents/examples/testing.adoc @@ -0,0 +1,294 @@ += Test MCP Examples +:description: Automated testing strategies for Redpanda Cloud MCP server examples. + +This document describes the automated testing strategies for Redpanda Cloud MCP server examples. + +All MCP examples are automatically tested to ensure: + +. YAML syntax and structure are correct +. MCP metadata is complete and valid +. Component schemas match Redpanda Connect specifications +. Secrets syntax uses Cloud Secrets Store format (`${secrets.X}`) + +== Testing approaches + +=== Configuration linting + +Validate MCP tool configurations using `rpk connect lint`: + +[,bash] +---- +# Lint a single MCP tool +rpk connect lint weather_service.yaml + +# Lint all examples +rpk connect lint *.yaml + +# Lint with environment variable checking skipped (recommended for MCP) +rpk connect lint --skip-env-var-check *.yaml +---- + +This checks for common issues such as: + +* YAML syntax errors +* Unknown component types +* Invalid field names +* Type mismatches +* Missing required fields + +=== MCP metadata validation + +The test script validates MCP-specific metadata for all tool examples: + +[,bash] +---- +# Run all tests (includes linting + MCP validation) +./test-mcp-examples.sh + +# Test specific files +./test-mcp-examples.sh weather_*.yaml +---- + +MCP metadata validation checks: + +* Presence of `meta.mcp` section +* `enabled: true` is set +* `description` field exists and is non-empty +* `properties` are properly structured (if present) + +=== Unit testing limitations + +[IMPORTANT] +==== +MCP tool examples are standalone component definitions (e.g., `label:`, `processors:`, `meta:`), not full pipelines with `input:`, `pipeline:`, `output:` sections. This means they cannot use inline `tests:` sections like cookbook examples do. + +The `rpk connect test` command requires full pipeline structure with paths like `/pipeline/processors/0`, which don't exist in MCP tool definitions. +==== + +For testing MCP tools: + +* **Linting is the primary validation** - ensures syntax and schema correctness +* **MCP metadata validation** - verifies tool has proper description and properties +* **Manual testing** - uses the Cloud Console MCP Server interface to test tools end-to-end + +== MCP tool structure + +MCP tools are structured as standalone components: + +[,yaml] +---- +label: weather-service +processors: + - label: fetch_weather_data + http: + url: 'https://wttr.in/${! @city }?format=j1' + verb: GET + + - label: format_response + mutation: | + root = { + "city": @city, + "temperature": this.current_condition.0.temp_C.number() + } + +meta: + mcp: + enabled: true + description: "Get current weather conditions for any city worldwide" + properties: + - name: city + type: string + description: "Name of the city" + required: true +---- + +== Test script usage + +The `test-mcp-examples.sh` script provides automated validation: + +[,bash] +---- +# Test all examples +./test-mcp-examples.sh + +# Test specific files +./test-mcp-examples.sh weather_*.yaml +./test-mcp-examples.sh customer_*.yaml +---- + +The script provides color-coded output: + +[,console] +---- +๐Ÿงช Redpanda Connect MCP Examples Test Suite (Cloud) +==================================================== + +๐Ÿ“„ Testing: weather_service.yaml + Linting weather_service.yaml... PASSED + Validating MCP metadata... PASSED + +==================================================== +๐Ÿ“Š Test Summary +==================================================== +Total configs tested: 10 +Passed: 10 +Failed: 0 + +โœ… All tests passed! +---- + +== Manual end-to-end testing + +For comprehensive validation, test MCP tools using the Cloud Console: + +. Navigate to your Cloud cluster's MCP Server configuration +. Add or update your MCP tool configuration +. Use the Cloud Console's AI chat interface to invoke the tool +. Verify the tool executes correctly and returns expected results + +This validates: + +* Tool loads correctly in the MCP server +* MCP metadata is properly exposed to AI clients +* Tool executes with provided parameters +* Responses are formatted correctly +* Secrets are properly resolved from Cloud Secrets Store + +== GitHub Actions CI/CD + +Automated tests run on every push and pull request using GitHub Actions. + +The workflow tests all examples whenever: + +* Any `.yaml` file in `modules/ai-agents/examples/` changes +* The test script itself is modified + +See `.github/workflows/test-mcp-examples.yaml` for the complete workflow. + +== Best practices + +=== Use descriptive tool names + +[,yaml] +---- +# Good +label: fetch-customer-orders + +# Bad +label: tool1 +---- + +=== Write clear MCP descriptions + +[,yaml] +---- +# Good +meta: + mcp: + description: "Fetch a customer's order history and calculate spending metrics over the last 30 days" + +# Bad +meta: + mcp: + description: "Get orders" +---- + +=== Document all properties + +[,yaml] +---- +# Good +properties: + - name: customer_id + type: string + description: "Unique identifier for the customer" + required: true + - name: days + type: number + description: "Number of days to look back (default: 30)" + required: false + +# Bad +properties: + - name: id + type: string + required: true +---- + +=== Use Cloud Secrets Store for sensitive data + +[,yaml] +---- +# Cloud format - uses Secrets Store +sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "customers" +---- + +=== Tag your examples + +[,yaml] +---- +meta: + tags: [ example, weather, api ] # Helps organize and filter + mcp: + enabled: true +---- + +== Adding new examples + +When adding new MCP tool examples: + +. **Create your YAML file** in `modules/ai-agents/examples/`: ++ +[,bash] +---- +cd modules/ai-agents/examples +touch my-new-tool.yaml +---- + +. **Include complete MCP metadata:** ++ +[,yaml] +---- +label: my-new-tool +processors: + # Your processor configuration + +meta: + mcp: + enabled: true + description: "Clear, task-oriented description" + properties: + - name: param_name + type: string + description: "Parameter purpose and constraints" + required: true +---- + +. **Lint your example:** ++ +[,bash] +---- +rpk connect lint --skip-env-var-check my-new-tool.yaml +---- + +. **Run automated tests:** ++ +[,bash] +---- +./test-mcp-examples.sh my-new-tool.yaml +---- + +. **Test in Cloud Console (recommended):** ++ +Deploy your MCP server configuration and test the tool through the Cloud Console AI interface. + +. **Commit your example:** ++ +[,bash] +---- +git add modules/ai-agents/examples/my-new-tool.yaml +git commit -m "Add my-new-tool MCP example" +---- diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc index 5c32a437f..731129252 100644 --- a/modules/ai-agents/pages/index.adoc +++ b/modules/ai-agents/pages/index.adoc @@ -1,5 +1,4 @@ = AI Agents in Redpanda Cloud -:page-beta: true :description: Learn about AI agents and the tools Redpanda Cloud provides for building them. :page-layout: index :page-aliases: develop:agents/about.adoc, develop:ai-agents/about.adoc @@ -8,6 +7,6 @@ AI agents are configurable assistants that autonomously perform specialist tasks Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: -**Local MCP server for Redpanda Cloud**: A built-in server that gives your AI assistant direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. +**Local MCP server for Redpanda Cloud** badge::[label=beta, tooltip={page-beta-text}]: A built-in server that gives your AI assistant direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. **Remote MCP (managed)**: Your own custom MCP servers built with Redpanda Connect pipelines and hosted inside your Redpanda Cloud cluster. These let you create reusable tools for your team that integrate with knowledge bases, APIs, and web content while running close to your data. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/configuration.adoc b/modules/ai-agents/pages/mcp/local/configuration.adoc index 7b4453f3c..4b3c4cee7 100644 --- a/modules/ai-agents/pages/mcp/local/configuration.adoc +++ b/modules/ai-agents/pages/mcp/local/configuration.adoc @@ -6,9 +6,9 @@ This page explains how to configure the local MCP server for Redpanda Cloud, inc == Prerequisites -* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] -* Access to a Redpanda Cloud account -* Supported AI client (Claude, Claude Code, or other MCP-compatible client) +* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer]. +* Access to a Redpanda Cloud account. +* Supported AI client (Claude, Claude Code, or other MCP-compatible client). == Endpoint availability diff --git a/modules/ai-agents/pages/mcp/local/quickstart.adoc b/modules/ai-agents/pages/mcp/local/quickstart.adoc index 6cf7ee111..cf5f7be3d 100644 --- a/modules/ai-agents/pages/mcp/local/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/local/quickstart.adoc @@ -8,9 +8,9 @@ If you're trying to deploy your own MCP server as a managed service inside your == Prerequisites -* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] -* Access to a Redpanda Cloud account -* link:https://support.anthropic.com/en/articles/10065433-installing-claude-desktop[Claude] or link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed +* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer]. +* Access to a Redpanda Cloud account. +* link:https://support.anthropic.com/en/articles/10065433-installing-claude-desktop[Claude] or link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed. + TIP: For other clients, see xref:ai-agents:mcp/local/configuration.adoc[]. diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 6024bfeae..05bcd3f79 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -1,9 +1,8 @@ = MCP in Redpanda Cloud -:page-beta: true :description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Redpanda Cloud MCP Server and Remote MCP (managed). -**Model Context Protocol (MCP)** is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. +Model Context Protocol (MCP) is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. == Why use MCP with Redpanda Cloud? @@ -15,7 +14,7 @@ In Redpanda Cloud, there are two complementary options to use MCP: [cols="1s,2a,2a"] |=== -| | Redpanda Cloud MCP Server | Remote MCP +| |Redpanda Cloud MCP Server badge::[label=beta, tooltip={page-beta-text}] | Remote MCP | Description | A server that gives your AI assistant access to your Redpanda Cloud account and clusters. diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc index ec09bc792..8df0ea460 100644 --- a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -1,56 +1,56 @@ -= Manage Remote MCP Servers -:page-beta: true -:description: Learn how to manage, edit, stop, resume, and delete Remote MCP servers in Redpanda Cloud. += Remote MCP Server Administration Guide +:description: Overview of administrative tasks for managing MCP servers in Redpanda Cloud. - -This guide is for Redpanda Cloud administrators who need to manage the lifecycle of Remote MCP servers. +This guide provides an overview of administrative tasks for MCP servers. Use these resources to manage your MCP servers throughout their lifecycle, from creation to deletion. == Prerequisites You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. -== Edit an existing MCP server +== Server lifecycle management -You can update the configuration, resources, or metadata of a Remote MCP server at any time. +Manage the basic lifecycle of your MCP servers, including editing configurations, pausing to save costs, and deleting. -. In the Redpanda Cloud Console, navigate to *Remote MCP*. -. Find the MCP server you want to edit and click its name. -. Click *Edit configuration*. -. Make your changes. -. Click *Save* to apply changes. +See xref:ai-agents:mcp/remote/manage-servers.adoc[]. -[NOTE] -==== -Editing a running MCP server may cause a brief interruption. Review changes before deploying to production. -==== +== Resource scaling -== Stop (pause) an MCP server +Adjust your MCP server's compute resources to match workload demands and optimize costs. Resource allocation directly affects your billing charges. -Pausing a server stops all tool execution and releases compute resources, but preserves configuration and state. +See xref:ai-agents:mcp/remote/scale-resources.adoc[]. -. In the *Remote MCP* list, find the server you want to stop. -. Click the three dots and select *Stop*. -. Confirm the action. +== Monitoring and observability -[NOTE] -==== -While stopped, the server does not respond to MCP requests. Start it again to restore service. -==== +Monitor your MCP server's activity using OpenTelemetry traces. Track tool invocations, measure performance, debug failures, and integrate with observability platforms. + +See xref:ai-agents:mcp/remote/monitor-activity.adoc[]. + +== Data Plane API -== Resume a stopped MCP server +All administrative tasks can be performed through both the Redpanda Cloud Console UI and the Data Plane API. The Data Plane API is useful for: -. In the *Remote MCP* list, find the stopped server. -. Click the three dots and select *Start*. -. Wait for the status to show *Running* before reconnecting clients. +* Automation and Infrastructure as Code (IaC) +* CI/CD pipeline integration +* Custom tooling and scripts +* Programmatic management at scale -== Delete an MCP server +To use the Data Plane API: -Deleting a server is permanent and cannot be undone. +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Use the appropriate endpoint for your task: ++ +* link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_updatemcpserver[`PATCH /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`]: Update configuration +* link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_stopmcpserver[`POST /v1/redpanda-connect/mcp-servers/\{mcp_server_id}:stop`]: Stop server +* link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_startmcpserver[`POST /v1/redpanda-connect/mcp-servers/\{mcp_server_id}:start`]: Start server +* link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_deletemcpserver[`DELETE /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`]: Delete server -. In the *Remote MCP* list, find the server to delete. -. Click the three dots and select *Delete*. -. Confirm the deletion when prompted. +[NOTE] +==== +Access tokens expire after 1 hour. To refresh, make the authentication request again with your service account credentials. The same token works for both Control Plane and Data Plane API requests. +==== -== API and automation +== Next steps -All of these actions can also be performed using the Redpanda Cloud API version v1alpha3. See the link:https://github.com/redpanda-data/console/blob/master/proto/gen/openapi/openapi.v1alpha3.yaml[OpenAPI specification file] for endpoints to manage MCP servers. \ No newline at end of file +* xref:ai-agents:mcp/remote/developer-guide.adoc[Build custom tools] for your specific use cases. +* xref:develop:connect/configuration/secret-management.adoc[Manage secrets] used by MCP server tools. +* xref:billing:billing.adoc#remote-mcp-billing-metrics[Understand MCP billing] to optimize costs. diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 5fc7affaf..0db151426 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -1,16 +1,15 @@ = Build Remote MCP Servers in Redpanda Cloud -:page-beta: true -:description: Learn how to write and deploy Remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. +:description: Learn how to write and deploy MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. -This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[Remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. +This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[MCP servers] that are managed in Redpanda Cloud. MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. == Prerequisites -* Access to a Redpanda Cloud cluster -* Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries -* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] -* (Optional) An AI assistant like Claude or Claude Code for testing +* Access to a Redpanda Cloud cluster. +* Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries. +* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer]. +* (Optional) An AI assistant like Claude or Claude Code for testing. TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. @@ -20,7 +19,7 @@ You should also be familiar with YAML, HTTP APIs, and basic event-stream process Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write each tool's logic using Redpanda Connect configuration and annotate them with MCP metadata so clients can discover and invoke them. -Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your Remote MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. +Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. Tool:: A single request/response operation exposed to MCP. @@ -28,26 +27,72 @@ Secrets:: Credentials and tokens that you want to use in your tools. These value == Development workflow -. <>: Select a Redpanda Cloud cluster to host your MCP server. +. <>: Navigate to Remote MCP in Redpanda Cloud Console, create a new server, and configure server metadata (display name, description, tags, resources, service account). . <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. . <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. -. <>: Build the logic for your MCP tool using Redpanda Connect configurations. +. <>: Build your MCP tool using Redpanda Connect components. . <>: Validate your configuration, catch schema errors, then deploy and test with the MCP Inspector. . <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. . <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. [[access]] -== Access MCP Servers in Redpanda Cloud +== Create MCP server and configure metadata . Log into your link:https://cloud.redpanda.com[Redpanda Cloud Console^]. . Select your cluster from the cluster list. . In the left sidebar, navigate to *Remote MCP*. . If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. -. Fill in the required fields: +. Click *Create MCP Server* and configure the *Server Metadata*: + -* *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). +* *Display Name*: A human-friendly name for your server (for example `weather-tool`, `customer-lookup`). * *Description*: Brief description of what your server does. -* *Configuration*: This is where to put your YAML configuration to define an MCP tool. +* *Tags*: Key-value pairs for organizing and categorizing your MCP servers. See <> for details. +* *Resources*: Choose the compute resources (XSmall through XLarge) based on your expected workload. +* *Service Account*: A service account name is pre-filled and will be created automatically. ++ +. Click *Next* to define your tools using YAML configuration. See <> to design your tool contract. + +[[tags]] +== Organize servers with tags + +Tags are key-value metadata that help you organize, categorize, and filter your MCP servers. Use tags to track ownership, environment, cost allocation, or any other organizational dimension relevant to your team. + +=== Common tagging strategies + +[cols="1,2,2"] +|=== +| Strategy | Description | Examples + +| Ownership +| Track which team or individual owns the MCP server. +| `owner=platform-team`, `team=data-engineering` + +| Environment +| Identify the deployment environment. +| `env=production`, `env=staging`, `env=development` + +| Cost allocation +| Allocate costs to specific departments or projects. +| `cost-center=marketing`, `project=analytics` + +| Purpose +| Describe the use case or data domain. +| `purpose=customer-data`, `use-case=real-time-alerts` +|=== + +=== Reserved tag names + +The following tag names are reserved and cannot be used. These tags are system-managed: + +* `service_account_id`: Automatically assigned to track the service account associated with the MCP server. +* `secret_id`: Automatically assigned to track the service account secret used by the MCP server. + +=== Best practices + +* Use consistent naming conventions across your organization (for example, always lowercase with hyphens or underscores). +* Document your tagging strategy so all team members use tags consistently. +* Use tags for filtering and searching MCP servers in the Cloud Console and API. +* Review and update tags periodically to keep them accurate as servers evolve. [[contract]] == Design the tool contract and MCP metadata @@ -97,7 +142,20 @@ After defining your tool contract, implement the logic to handle input validatio [[provision-secrets]] == Provision secrets -All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Follow these best practices: +All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. + +=== Creating secrets in the Cloud Console + +When you reference secrets in your tool's YAML configuration using `${secrets.SECRET_NAME}`, the Cloud Console automatically detects these references and provides an interface next to the configuration editor where you can: + +. Enter the secret values directly +. Create the secrets in the Secrets Store with a single click + +This inline secret creation workflow simplifies the development process by letting you configure secrets as you build your tools, without navigating away to the Secrets Store. + +Alternatively, you can manually create and manage secrets directly in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. + +=== Best practices - Reference secrets as `${secrets.NAME}` in your YAML configuration. - Never commit secrets to Git or reference them directly inline in configuration files. @@ -108,7 +166,7 @@ All credentials and sensitive values must be stored in the Redpanda Cloud xref:d See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[MCP patterns]. [[pipeline-patterns]] -== Implement the logic of your MCP tool +== Build your MCP tool Use Redpanda Connect components to implement the logic of your MCP tool. Here are some best practices: @@ -277,7 +335,7 @@ See xref:ai-agents:mcp/remote/admin-guide.adoc[]. [[test]] == Authenticate and connect your MCP client -To connect your local MCP client to your Remote MCP server: +To connect your local MCP client to your MCP server: . First, authenticate to Redpanda Cloud: + @@ -311,15 +369,15 @@ rpk cloud mcp proxy \ ---- + -- -<1> The target Redpanda Cloud cluster ID where your Remote MCP server is hosted. -<2> The unique ID of your deployed Remote MCP server +<1> The target Redpanda Cloud cluster ID where your MCP server is hosted. +<2> The unique ID of your deployed MCP server -- + TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. - -The proxy acts as a bridge between your local client and the Remote MCP server running in your cluster. It: - -* Connects to your Remote MCP server using your authentication token ++ +The proxy acts as a bridge between your local client and the MCP server running in your cluster. It: ++ +* Connects to your MCP server using your authentication token * Discovers and registers all tools from the Remote server locally * Proxies MCP requests from your client to the Remote server * Handles authentication by injecting your token into each request @@ -330,7 +388,7 @@ The proxy acts as a bridge between your local client and the Remote MCP server r [NOTE] ==== -You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement of Remote MCP. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. +You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement for MCP servers. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. ==== :note-caption: Note diff --git a/modules/ai-agents/pages/mcp/remote/index.adoc b/modules/ai-agents/pages/mcp/remote/index.adoc index 43fa14dd4..39771ad81 100644 --- a/modules/ai-agents/pages/mcp/remote/index.adoc +++ b/modules/ai-agents/pages/mcp/remote/index.adoc @@ -1,4 +1,3 @@ = Remote MCP Servers for Redpanda Cloud -:page-beta: true :description: Enable AI assistants to directly interact with your Redpanda Cloud clusters and streaming data. :page-layout: index diff --git a/modules/ai-agents/pages/mcp/remote/manage-servers.adoc b/modules/ai-agents/pages/mcp/remote/manage-servers.adoc new file mode 100644 index 000000000..e0aeb6a29 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/manage-servers.adoc @@ -0,0 +1,154 @@ += Manage Remote MCP Servers +:description: Learn how to edit, stop, start, and delete MCP servers in Redpanda Cloud. + +This guide shows you how to manage the lifecycle of MCP servers, including editing configurations, pausing servers to save costs, and permanent deletion. + +== Prerequisites + +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +== Edit an MCP server + +You can update the configuration, resources, or metadata of an MCP server at any time. + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the MCP server you want to edit and click its name. +. Click *Edit configuration*. +. Make your changes. +. Click *Save* to apply changes. + +[NOTE] +==== +Editing a running MCP server may cause a brief interruption. Review changes before deploying to production. +==== +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_getmcpserver[`GET /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`] to retrieve the current configuration. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_updatemcpserver[`PATCH /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`] to update the configuration: ++ +[,bash] +---- +curl -X PATCH "https:///v1/redpanda-connect/mcp-servers/?update_mask=display_name,description" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "mcp_server": { + "display_name": "updated-name", + "description": "Updated description" + } + }' +---- +-- +===== + +== Stop an MCP server + +Stopping a server pauses all tool execution and releases compute resources, but preserves configuration and state. This is useful for temporarily disabling a server to save costs while retaining the ability to restart it later. + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the server you want to stop. +. Click the three dots and select *Stop*. +. Confirm the action. +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_stopmcpserver[`POST /v1/redpanda-connect/mcp-servers/\{mcp_server_id}:stop`]: ++ +[,bash] +---- +curl -X POST "https:///v1/redpanda-connect/mcp-servers/:stop" \ + -H "Authorization: Bearer " +---- +-- +===== + + +While stopped, the server does not respond to MCP requests. Start it again to restore service. + +== Start a stopped MCP server + +Resume a stopped server to restore its functionality. + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the stopped server. +. Click the three dots and select *Start*. +. Wait for the status to show *Running* before reconnecting clients. +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_startmcpserver[`POST /v1/redpanda-connect/mcp-servers/\{mcp_server_id}:start`]: ++ +[,bash] +---- +curl -X POST "https:///v1/redpanda-connect/mcp-servers/:start" \ + -H "Authorization: Bearer " +---- ++ +. Wait for the status to show *Running* before reconnecting clients. +-- +===== + +== Delete an MCP server + +Deleting a server is permanent and cannot be undone. All configuration, tools, and associated resources are removed. + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the server you want to delete. +. Click the three dots and select *Delete*. +. Confirm the deletion when prompted. +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_deletemcpserver[`DELETE /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`]: ++ +[,bash] +---- +curl -X DELETE "https:///v1/redpanda-connect/mcp-servers/" \ + -H "Authorization: Bearer " +---- +-- +===== + +[WARNING] +==== +Deletion is immediate and permanent. Make sure you have backed up any important configuration before deleting an MCP server. +==== + +== Next steps + +* xref:ai-agents:mcp/remote/scale-resources.adoc[Scale MCP server resources] to optimize performance and costs. +* xref:ai-agents:mcp/remote/monitor-activity.adoc[Monitor MCP server activity] using OpenTelemetry traces. +* xref:ai-agents:mcp/remote/developer-guide.adoc[Build custom tools] for your specific use cases. diff --git a/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc b/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc new file mode 100644 index 000000000..fca49c323 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc @@ -0,0 +1,296 @@ += Monitor Remote MCP Server Activity +:description: Learn how to monitor MCP servers using OpenTelemetry traces, track tool invocations, debug failures, and integrate with observability platforms. + +Every MCP server automatically emits OpenTelemetry traces to a topic called `redpanda.otel_traces`. These traces provide detailed observability into your MCP server's operations, including tool invocations, processing times, errors, and resource usage. + +== Prerequisites + +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +== Execution log + +When you create an MCP server, Redpanda captures a complete execution log by recording all operations as OpenTelemetry traces with 100% sampling. This execution log is stored in the `redpanda.otel_traces` topic. + +=== Traces and spans + +OpenTelemetry traces provide a complete picture of how a request flows through your system: + +* A *trace* represents the entire lifecycle of a request (for example, a tool invocation from start to finish). +* A *span* represents a single unit of work within that trace (such as a data processing operation or an external API call). +* A trace contains one or more spans organized hierarchically, showing how operations relate to each other. + +With 100% sampling, every operation is captured, creating a complete execution log that you can use for debugging, monitoring, and performance analysis. + +=== How Redpanda stores traces + +The `redpanda.otel_traces` topic stores OpenTelemetry spans in JSON format, following the https://opentelemetry.io/docs/specs/otel/protocol/[OpenTelemetry Protocol (OTLP)^] specification. A Protobuf schema named `redpanda.otel_traces-value` is also automatically registered with the topic, enabling clients to deserialize trace data correctly. + +The `redpanda.otel_traces` topic and its schema are managed automatically by Redpanda. If you delete either the topic or the schema, they are recreated automatically. However, deleting the topic permanently deletes all trace data, and the topic comes back empty. Do not produce your own data to this topic. It is reserved for OpenTelemetry traces. + +Each span in the execution log represents a specific operation performed by your MCP server, such as: + +* Tool invocation requests +* Data processing operations +* External API calls +* Error conditions +* Performance metrics + +=== Topic configuration and lifecycle + +The `redpanda.otel_traces` topic has a predefined retention policy. Configuration changes to this topic are not supported. If you modify settings, Redpanda reverts them to the default values. + +The topic persists in your cluster even after all MCP servers are deleted, allowing you to retain historical trace data for analysis. + +You can create glossterm:ACL[,access control lists (ACLs)] to control access to the `redpanda.otel_traces` topic using standard Redpanda ACL operations. + +Trace data may contain sensitive information from your tool inputs and outputs. Consider implementing appropriate access controls and review the data in traces before sharing or exporting to external systems. + +=== Traces compared to audit logs + +OpenTelemetry traces are designed for observability and debugging, not audit logging or compliance: + +Traces provide: + +* Hierarchical view of request flow through your system (parent-child span relationships) +* Detailed timing information for performance analysis +* Ability to reconstruct execution paths and identify bottlenecks +* Insights into how operations flow through distributed systems + +Traces are not: + +* Immutable audit records for compliance purposes +* Designed for "who did what" accountability tracking + +== Consume traces from the execution log + +You can consume traces from the `redpanda.otel_traces` topic using any Kafka-compatible client or the Redpanda Console. + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Topics*. +. Select `redpanda.otel_traces`. +. Click *Messages* to view recent traces. +. Use filters to search for specific trace IDs, span names, or time ranges. +-- + +rpk:: ++ +-- +Consume the most recent traces: + +[,bash] +---- +rpk topic consume redpanda.otel_traces --offset end -n 10 +---- + +Filter for specific MCP server activity by examining the span attributes. +-- + +Data Plane API:: ++ +-- +Use the link:/api/doc/cloud-dataplane/topic/topic-quickstart[Data Plane API] to programmatically consume traces and integrate with your monitoring pipeline. +-- +===== + +== Trace data structure + +Each trace message contains an OpenTelemetry span with the following structure: + +[,json] +---- +{ + "traceId": "71cad555b35602fbb35f035d6114db54", + "spanId": "43ad6bc31a826afd", + "parentSpanId": "", + "name": "http_processor", + "kind": 1, + "startTimeUnixNano": "1765198415253280028", + "endTimeUnixNano": "1765198424660663434", + "attributes": [ + { + "key": "city_name", + "value": {"stringValue": "london"} + }, + { + "key": "result_length", + "value": {"intValue": "198"} + } + ], + "status": { + "code": 0, + "message": "" + }, + "instrumentationScope": { + "name": "rpcn-mcp" + }, + "flags": 1 +} +---- + +* `traceId`: Unique identifier for the entire trace (hex-encoded string). +* `spanId`: Unique identifier for this specific span (hex-encoded string). +* `parentSpanId`: Links child spans to parent operations (empty string if root span). +* `name`: Descriptive name of the operation (typically the tool name in the case of MCP servers). +* `kind`: Span kind as integer (see the `redpanda.otel_traces-value` schema for details). +* `startTimeUnixNano` / `endTimeUnixNano`: Timing information in nanoseconds since Unix epoch. +* `attributes`: Array of key-value pairs with contextual information about the operation. +* `status`: Operation status with `code` (0 = OK, 2 = ERROR) and optional `message`. +* `instrumentationScope`: Identifies the library or component that created the span. + +=== Identifying MCP servers and tools + +All MCP servers in a cluster write traces to the same `redpanda.otel_traces` topic. Each trace has a unique `traceId`, and you can identify which tool created the trace by examining the span fields: + +* When `instrumentationScope.name` is `rpcn-mcp`, the span represents an MCP tool invocation. +* The `name` field contains the MCP tool name (for example, `weather`, `http_processor`, `generate_input`). +* Child spans reference their parent using `parentSpanId` and share the same `traceId` + +For example, a span with `instrumentationScope.name: "rpcn-mcp"` and `name: "weather"` indicates the `weather` tool was invoked. Child spans created by Redpanda Connect components show `instrumentationScope.name: "redpanda-connect"` and represent internal processing steps. + +=== Real trace examples + +Here are actual traces from MCP tool executions showing the parent-child span relationships and detailed attributes. + +**Successful tool execution:** + +This trace shows an `http_processor` tool that fetched weather data for London. The parent span includes child spans for mutations, HTTP requests, and processing: + +[source,json] +---- +{ + "attributes": [ + { + "key": "city_name", + "value": {"stringValue": "london"} + }, + { + "key": "result_prefix", + "value": {"stringValue": "{\"city\":\"london\",\"description\":\"Partly cloudy\",\"feels_like\":12,\"humidity\":88,\"metadata\":{\"fetched_at\":\"2025-12-08T12:53:44.660Z\""} + }, + { + "key": "result_length", + "value": {"intValue": "198"} + } + ], + "endTimeUnixNano": "1765198424660663434", + "flags": 1, + "instrumentationScope": {"name": "rpcn-mcp"}, + "kind": 1, + "name": "http_processor", + "spanId": "43ad6bc31a826afd", + "startTimeUnixNano": "1765198415253280028", + "status": {"code": 0, "message": ""}, + "traceId": "71cad555b35602fbb35f035d6114db54" +} +---- + +The parent span (`http_processor`) has child spans showing the processing pipeline: + +[,json] +---- +{ + "attributes": [], + "endTimeUnixNano": "1765198424659619511", + "flags": 1, + "instrumentationScope": {"name": "redpanda-connect"}, + "kind": 1, + "name": "http", + "parentSpanId": "43ad6bc31a826afd", + "spanId": "ed45544a7d7b08d4", + "startTimeUnixNano": "1765198415253319496", + "status": {"code": 0, "message": ""}, + "traceId": "71cad555b35602fbb35f035d6114db54" +} +---- + +* The parent span (`http_processor`) includes custom attributes like `city_name` and `result_length` that help debug and monitor tool behavior. +* Child spans use the same `traceId` but different `spanId` values. +* The `parentSpanId` in child spans matches the parent's `spanId`, creating the hierarchy. +* Duration can be calculated: `(1765198424660663434 - 1765198415253280028) / 1000000 = 9407ms`. + +**Trace with error event:** + +This trace shows an HTTP request that encountered an error during execution: + +[,json] +---- +{ + "attributes": [], + "endTimeUnixNano": "1765198423080903265", + "events": [ + { + "attributes": [ + { + "key": "error", + "value": {"stringValue": "type"} + } + ], + "name": "event", + "timeUnixNano": "1765198420254169629" + } + ], + "flags": 1, + "instrumentationScope": {"name": "redpanda-connect"}, + "kind": 1, + "name": "http_request", + "parentSpanId": "43ad6bc31a826afd", + "spanId": "ba332199f3af6d7f", + "startTimeUnixNano": "1765198415253325782", + "status": {"code": 0, "message": ""}, + "traceId": "71cad555b35602fbb35f035d6114db54" +} +---- + +* The span still shows `status.code: 0` (success) overall. +* However, the `events` array contains error information with `error: type`. +* This indicates a recoverable error or warning during execution. +* The event's `timeUnixNano` shows exactly when the error occurred. + +== Common monitoring scenarios + +=== Track tool invocations + +Monitor which tools are being called and how often: + +. Consume traces from `redpanda.otel_traces`. +. Filter spans where `name` contains `tool_invocation`. +. Examine the `mcp.tool.name` attribute to see which tools are being used. +. Calculate frequency by counting spans per tool name over time windows. + +=== Measure performance + +Analyze tool execution times: + +. Find spans with `mcp.tool.name` attributes. +. Calculate duration: `(endTimeUnixNano - startTimeUnixNano) / 1000000` (milliseconds). +. Track percentiles (p50, p95, p99) to identify performance issues. +. Set alerts for durations exceeding acceptable thresholds. + +=== Debug failures + +Investigate errors and failures: + +. Filter spans where `status.code` is not `STATUS_CODE_OK`. +. Look for `STATUS_CODE_ERROR` in the status field. +. Examine `status.message` for error details. +. Use `traceId` to correlate related spans and understand the full error context. + +=== Correlate distributed operations + +Link MCP server activity to downstream effects: + +. Extract `traceId` from tool invocation spans. +. Search for the same `traceId` in other application logs or traces. +. Follow `parentSpanId` relationships to build complete operation timelines. +. Identify bottlenecks across your entire system. + +== Integration with observability platforms + +The `redpanda.otel_traces` topic stores trace data in OpenTelemetry format. Direct export to platforms like Grafana Cloud and Datadog is not supported due to format compatibility limitations. Redpanda produces one span per topic message, whereas these platforms expect traces in batch format. + +You can consume traces directly from the `redpanda.otel_traces` topic using any Kafka-compatible consumer for custom analysis and processing. + diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index ee5fc6dd7..2b66d6135 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -1,47 +1,21 @@ = About Remote MCP Servers for Redpanda Cloud -:page-beta: true :description: Discover how AI assistants can interact with your streaming data and how to connect them to Redpanda Cloud. - -Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that you can write and run inside your Redpanda Cloud cluster. They let you expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: +Remote MCP (Model Context Protocol) allows you to build and host MCP servers in your Redpanda Cloud clusters. They can expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: * Make your data and workflows accessible to AI assistants and LLMs, safely and on your terms. * Accelerate automation and insight by letting AI trigger real business actions. * Control exposure so that only the tools you tag are visible to AI. -== Why use a Remote MCP server? +== Why use Remote MCP? * *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. * *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. * *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in your YAML configuration. * *Fast iteration*: Define tools as YAML configuration, deploy your MCP server, and your AI agents can use the new logic instantly. -== Use cases - -[cols="1s,3a"] -|=== -|Category |Example prompts - -|Operational monitoring -|* Check partition lag for customer-events topic -* Show me the top 10 producers by message volume today -* Get schema registry health status - -|Data enrichment and analysis -|* Fetch user profile data and recent orders for customer ID 12345 -* Get real-time stock prices for symbols in my portfolio topic -* Analyze sentiment of latest product reviews - -|Team productivity -|* Deploy my microservice to the staging environment -* Generate load test data for the payments service -* Create a summary dashboard of this week's incident reports - -|Business intelligence -|* What are the trending products in the last 24 hours? -* Show revenue impact of the latest feature deployment -* Get customer satisfaction scores from support tickets -|=== +// Use cases table - single-sourced from rp-connect-docs partial +include::redpanda-connect:ai-agents:partial$mcp/use-cases-table.adoc[] == How it works @@ -49,44 +23,8 @@ Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that . Deploy to your cluster. Redpanda Cloud hosts your MCP server with secure access. . Connect your AI client. AI assistants authenticate, discover available tools, and invoke them using natural language prompts. -== Example: Customer analytics tool - -Here's how you might expose a customer analytics pipeline as an MCP tool: - -[source,yaml] ----- -label: analyze_customer_orders -processors: - - label: fetch_customer_data - sql_select: - driver: "postgres" - dsn: "${secrets.POSTGRES_DSN}" - table: "orders" - where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" - args_mapping: 'root = [this.customer_id]' - - - label: calculate_metrics - mutation: | - root = { - "customer_id": this.customer_id, - "total_orders": this.length(), - "total_spent": this.map_each(o -> o.total).sum(), - "avg_order_value": this.map_each(o -> o.total).mean(), - "last_order_date": this.map_each(o -> o.created_at).max() - } - -meta: - mcp: - enabled: true - description: "Analyze a customer's order history and spending patterns over the last 30 days" - properties: - - name: customer_id - type: string - description: "Customer ID to analyze" - required: true ----- - -This tool can be invoked by an AI assistant with prompts like "analyze the order history for customer ID 12345" without needing to know the underlying SQL or processing logic. +// Customer analytics example - single-sourced from rp-connect-docs partial +include::redpanda-connect:ai-agents:partial$mcp/customer-analytics-example.adoc[] == Authentication options @@ -98,13 +36,10 @@ rpk cloud login rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id ---- -- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the Remote MCP server. - -== MCP specification support - -Remote MCP server implements the open MCP protocol for tool exposure. Only the tool concept from the MCP server specification is supported. Features such as MCP resources and prompts are not yet available. +- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the MCP server. -For full details, see the official MCP server specification: link:https://modelcontextprotocol.io/specification/2025-06-18/server[modelcontextprotocol.io/specification/2025-06-18/server^]. +// MCP specification support - single-sourced from rp-connect-docs partial +include::redpanda-connect:ai-agents:partial$mcp/specification-support.adoc[] == Get started diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc index 70d589ece..72b7a9d1b 100644 --- a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc +++ b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc @@ -1,17 +1,33 @@ = Patterns for Remote MCP Servers -:description: Catalog of patterns for Remote MCP server tools in Redpanda Cloud. -:page-beta: true +:description: Catalog of patterns for MCP server tools in Redpanda Cloud. -This page provides a reference catalog of patterns designed for use with Remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. +This page provides a reference catalog of patterns designed for use with MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. For a high-level overview of MCP servers, see xref:ai-agents:mcp/overview.adoc[]. +== Pattern selection guide + +Choose the right pattern for your use case: + +* **Generate test data or synthetic events**: <> +* **Call external REST APIs**: <> +* **Query databases**: <> +* **Read from Redpanda topics**: <> +* **Transform or validate data**: <> +* **Publish to Redpanda topics**: <> +* **Integrate with AI/LLM services**: <> +* **Cache frequently accessed data**: <> + == Data generators Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. +**When to use:** Development and testing environments where you need synthetic data, load testing scenarios, or demonstrating data flows without real data sources. + +**Example use cases:** Mock user events, test order data, synthetic sensor readings, demo data for presentations. + This example generates a realistic user event message: [source,yaml] @@ -25,6 +41,10 @@ See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. +**When to use:** Integrating with third-party services, fetching real-time data, calling internal microservices, or enriching event data with external information. + +**Example use cases:** Fetch user profile from CRM, get product pricing from inventory API, validate addresses with geocoding service, retrieve weather data. + [source,yaml] ---- include::ai-agents:example$http_processor.yaml[] @@ -36,6 +56,10 @@ See also: xref:develop:connect/components/processors/http.adoc[`http` processor] Query external databases and return structured results. This pattern is essential for tools that need to access business data. +**When to use:** Retrieving customer records, querying analytics data, looking up configuration values, or joining streaming data with dimensional data from data warehouses. + +**Example use cases:** Fetch customer details from PostgreSQL, query sales data from BigQuery, retrieve product catalog from MongoDB, look up reference data. + [source,yaml] ---- include::ai-agents:example$gcp_bigquery_select_processor.yaml[] @@ -43,261 +67,164 @@ include::ai-agents:example$gcp_bigquery_select_processor.yaml[] See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) -=== Redpanda integration and data publishing +== Data consumers -Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. +Read messages from Redpanda topics to process streaming data in your MCP tools. This pattern enables event-driven workflows and real-time data processing. -=== Data publishers (output patterns) +**When to use:** Processing events from Redpanda topics, building event-driven AI agents, consuming audit logs, or subscribing to data change streams. -Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems or build caching systems. - -This example publishes a message to a Redpanda topic: +**Example use cases:** Monitor order events, process user activity streams, consume IoT sensor data, react to system notifications. [source,yaml] ---- -include::ai-agents:example$redpanda_output.yaml[] +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "user-events" ] + consumer_group: "mcp-event-processor" + start_from_oldest: true + tls: + enabled: true + sasl: + - mechanism: "${REDPANDA_SASL_MECHANISM}" + username: "${REDPANDA_SASL_USERNAME}" + password: "${REDPANDA_SASL_PASSWORD}" ---- -See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] +See also: xref:develop:connect/components/inputs/redpanda.adoc[`redpanda` input] -=== Caching systems +== Data transformation -Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. +Transform, validate, and enrich data as it flows through your MCP tools. Use Bloblang mapping language for powerful data manipulation. -[source,yaml] ----- -include::ai-agents:example$redpanda_cache.yaml[] ----- +**When to use:** Converting data formats, validating schemas, filtering events, enriching messages with computed fields, or normalizing data structures. -This example implements an in-memory cache for low-latency access to small datasets: +**Example use cases:** Parse JSON payloads, validate required fields, add timestamps, convert units, mask sensitive data, aggregate nested objects. [source,yaml] ---- -include::ai-agents:example$memory_cache.yaml[] ----- +mapping: | + # Parse and validate incoming data + root.user_id = this.user_id.or(throw("user_id is required")) + root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") -See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + # Transform and enrich + root.email_domain = this.email.split("@").index(1) + root.is_premium = this.subscription_tier == "premium" -== Production workflows and observability + # Filter sensitive data + root.profile = this.profile.without("ssn", "credit_card") +---- -Build enterprise-grade tools with error handling, validation, multi-step workflows, and monitoring. +See also: xref:develop:connect/components/processors/mapping.adoc[`mapping` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] -=== Parameter validation and type coercion +== AI/LLM integration -Always validate and coerce input parameters to ensure your tools are robust: +Integrate AI and LLM services into your MCP tools for intelligent data processing, embeddings generation, and natural language understanding. -[source,yaml] ----- -processors: - - label: validate_params - mutation: | - # Validate required parameters - root = if !this.exists("user_id") { - throw("user_id parameter is required") - } else { this } - - # Type coercion with validation - meta user_id = this.user_id.string() - meta limit = this.limit.number().catch(10) - meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) ----- +**When to use:** Generating embeddings for semantic search, calling LLM APIs for text generation, building RAG (Retrieval Augmented Generation) pipelines, or analyzing sentiment. -=== Dynamic configuration +**Example use cases:** Generate embeddings for documents, classify customer feedback, summarize long text, extract entities, answer questions with context. -Build tools that adapt their behavior based on input parameters: +=== OpenAI integration [source,yaml] ---- -processors: - - label: dynamic_config - mutation: | - # Choose data source based on environment - meta env = this.environment | "production" - meta table_name = match @env { - "dev" => "dev_orders", - "staging" => "staging_orders", - "production" => "prod_orders", - _ => "dev_orders" - } - - # Adjust query complexity based on urgency - meta columns = if this.detailed.bool().catch(false) { - ["order_id", "customer_id", "total", "items", "shipping_address"] - } else { - ["order_id", "customer_id", "total"] - } +openai_chat_completion: + api_key: "${secrets.OPENAI_API_KEY}" + model: "gpt-4" + prompt: | + Analyze this customer feedback and provide: + 1. Sentiment (positive/negative/neutral) + 2. Key themes + 3. Actionable insights + + Feedback: ${! json("feedback_text") } + max_tokens: 500 ---- -=== Error handling and fallbacks +See also: xref:develop:connect/components/processors/openai_chat_completion.adoc[`openai_chat_completion`], xref:develop:connect/components/processors/openai_embeddings.adoc[`openai_embeddings`] -Implement error handling to make your tools reliable: +=== Embeddings generation [source,yaml] ---- -processors: - - label: primary_fetch - try: - - http: - url: "https://api.primary.com/data" - timeout: "10s" - catch: - - log: - message: "Primary API failed, trying fallback" - - label: fallback_fetch - http: - url: "https://api.fallback.com/data" - timeout: "15s" - - mutation: | - root.metadata.source = "fallback" - root.metadata.warning = "Primary source unavailable" +openai_embeddings: + api_key: "${secrets.OPENAI_API_KEY}" + model: "text-embedding-3-small" + text: ${! json("content") } ---- -=== Conditional processing +See also: xref:develop:connect/components/processors/cohere_embeddings.adoc[`cohere_embeddings`], xref:develop:connect/components/processors/gcp_vertex_ai_embeddings.adoc[`gcp_vertex_ai_embeddings`] -Build tools that branch based on input or data characteristics: +== Redpanda integration -[source,yaml] ----- -processors: - - label: conditional_processing - switch: - - check: this.data_type == "json" - processors: - - json: - operator: "parse" - - mutation: 'root.parsed_data = this' - - check: this.data_type == "csv" - processors: - - csv: - parse: true - - mutation: 'root.parsed_data = this' - - processors: - - mutation: 'root.error = "Unsupported data type"' ----- +Publish data to Redpanda topics and implement caching systems backed by Redpanda. -[[secrets]] -=== Secrets and credentials +Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. -Securely handle multiple credentials and API keys. +=== Data publishers -Here is an example of using an API key secret. +Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems, publish events to Redpanda topics, or integrate with downstream services. -. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. +**When to use:** Publishing events to Redpanda for consumption by other services, creating event sourcing patterns, building audit trails, or triggering downstream workflows. -. Reference the secret in your YAML configuration: -+ -[source,yaml] ----- -processors: - - label: call_external_api - http: - url: "https://api.example.com/data" - verb: GET - headers: - Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> - Accept: "application/json" ----- -+ -<1> The secret is injected at runtime. Never store the actual API key in your YAML configuration. The actual secret value never appears in your configuration files or logs. - -=== Monitoring, debugging, and observability +**Example use cases:** Publish order confirmations, emit audit events, trigger notifications, create event-driven workflows. -Use structured logging, request tracing, and performance metrics to gain insights into tool execution. +This example publishes a message to a Redpanda topic: [source,yaml] ---- -include::ai-agents:example$observable_tool.yaml[] +include::ai-agents:example$redpanda_output.yaml[] ---- -Observability features: - -* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing -* *Execution timing*: Track how long your tools take to execute using nanosecond precision -* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` -* *Request/response metadata*: Log input parameters and response characteristics -* *Success tracking*: Monitor whether operations complete successfully - -You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: +See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] -[source,json] ----- -{ - "metadata": { - "execution_time_ms": 0.158977, - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "success": true, - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool" - }, - "trace": { - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool", - "version": "1.0.0" - }, - "user_id": "1" -} ----- +==== Outputs with processors -=== Multi-step data enrichment +Output tools can include processors to transform data before publishing. This pattern is useful when you need to process data and save the result to a destination in a single tool. -Build tools that combine data from multiple sources. +**When to use:** Processing user input with an LLM and saving the response, transforming data before publishing to a topic, enriching events before writing to external systems. -This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. +**Example use cases:** Send a prompt to an LLM, then save the answer to a topic in Redpanda. [source,yaml] ---- -include::ai-agents:example$customer_enrichment.yaml[] +include::ai-agents:example$redpanda_output_with_processors.yaml[] ---- -See also: xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) +You can use an output component type with processors, but you cannot use a processor component type with outputs. The `processors` field is available in all output components. -=== Workflow orchestration +=== Caching systems -Coordinate complex workflows with multiple steps and conditional logic. +Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. + +**When to use:** Reducing repeated API calls, storing lookup tables, caching database query results, or maintaining session state across tool invocations. -This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. +**Example use cases:** Cache user profiles, store API rate limit counters, maintain configuration values, cache product catalogs. +.Redpanda-backed cache [source,yaml] ---- -include::ai-agents:example$order_workflow.yaml[] +include::ai-agents:example$redpanda_cache.yaml[] ---- -For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: - -[source,json] +.In-memory cache for low-latency access to small datasets +[source,yaml] ---- -{ - "assigned_rep": "vip-team@company.com", - "available_quantity": 100, - "customer_tier": "vip", - "estimated_fulfillment": "TBD - calculated based on processing tier", - "inventory_check": "passed", - "order_id": "ORD001", - "order_status": "processed", - "perks": [ - "expedited_shipping", - "white_glove_service" - ], - "priority_score": 90, - "processed_at": "2025-09-16T09:05:29.138Z", - "processing_tier": "vip", - "processing_time_estimate": "1-2 hours", - "processing_time_hours": 2, - "product_id": "widget-001", - "product_name": "Standard Widget", - "quantity": 5, - "total": 250 -} +include::ai-agents:example$memory_cache.yaml[] ---- -Notice how the workflow: +See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +// Production workflows section - single-sourced from rp-connect-docs partials +// Part 1: Parameter validation, dynamic config, error handling, conditional processing +include::redpanda-connect:ai-agents:partial$mcp/production-workflows-before-secrets.adoc[] + +// Secrets section - deployment-specific for Cloud (uses Secrets Store) +include::redpanda-connect:ai-agents:partial$mcp/secrets-cloud.adoc[] -. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. -. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. -. Routes by customer tier: Since `customer_tier` is "vip", it gets VIP processing with special `perks` and priority. -. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. -. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. +// Part 2: Monitoring, multi-step enrichment, workflow orchestration +include::redpanda-connect:ai-agents:partial$mcp/production-workflows-after-secrets.adoc[] == Suggested reading diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 6fe1de768..a43e10efa 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -1,23 +1,48 @@ = Remote MCP Server Quickstart -:page-beta: true -:description: Create and host a managed MCP server inside your Redpanda Cloud cluster with tools for generating and publishing user event data. +:description: Learn how to extend AI agents with custom tools that interact with your Redpanda data using the Model Context Protocol (MCP). +Model Context Protocol (MCP) enables AI assistants like Claude to interact with external systems through custom tools. This quickstart teaches you how to create an MCP server in Redpanda Cloud with tools that generate and publish event data, then connect Claude Code to use those tools through natural language commands. -This quickstart guide shows you how to create and host a managed MCP server inside your Redpanda Cloud cluster. The server includes two tools: one for generating user event data and another for publishing that data to a Redpanda topic. +== What you'll learn + +By completing this quickstart, you'll understand: + +* What MCP is and how it extends AI capabilities +* How MCP servers, tools, and clients work together +* How to create and deploy custom tools in Redpanda Cloud +* How to connect AI assistants to your Redpanda data infrastructure + +== How MCP works + +The Model Context Protocol defines a standard way for AI assistants to interact with external tools and data sources: + +MCP Server:: Hosts a collection of tools that perform specific actions. MCP servers run as managed services in your Redpanda Cloud cluster. +Tools:: Individual capabilities your AI can use, built with Redpanda Connect components. Tools can read from sources (inputs), transform data (processors), write to destinations (outputs), or cache results. +Client:: An AI assistant (like Claude Code) that discovers available tools and calls them based on user requests. + +In this quickstart, you'll create a server with two tools: + +. `generate_input`: Creates fake user event data with randomized fields +. `redpanda_output`: Publishes generated events to a Redpanda topic + +Once connected, you can ask Claude in natural language to "generate 10 events and publish them to Redpanda," and Claude will automatically call your tools to complete the task. == Prerequisites -* A Redpanda Cloud cluster with *Remote MCP* enabled +* A Redpanda Cloud cluster with *Remote MCP* enabled. * Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials. -* At least version 25.2.5 of the xref:manage:rpk/rpk-install.adoc[Redpanda CLI (`rpk`)] installed on your computer -* link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed - -TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. +* At least version 25.2.5 of the xref:manage:rpk/rpk-install.adoc[Redpanda CLI (`rpk`)] installed on your computer. +* link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed. == Prepare your cluster -Before creating the MCP server, you need to set up a topic and user for event publishing. Use the Redpanda CLI to perform these steps. +Before creating the MCP server, you need to set up a topic for event publishing. +[tabs] +====== +rpk:: ++ +-- . Log in to your Redpanda Cloud account: + [,bash] @@ -33,42 +58,107 @@ This opens a browser window to authenticate. The token is saved locally inside y ---- rpk topic create events --partitions 3 --replicas 3 ---- +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-controlplane/authentication#topic-request-an-access-token[Authenticate to the Control Plane API] to get an access token. ++ +[NOTE] +==== +Access tokens expire after 1 hour. To refresh, make the same authentication request again with your service account credentials. The same token works for both Control Plane and Data Plane API requests. +==== -. Create a user called `mcp` with a strong password: +. Get the Data Plane API URL for your cluster: ++ +[tabs] +==== +BYOC or Dedicated:: ++ +Make a request to the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_getcluster[`GET /v1/clusters/\{id}`] endpoint: ++ +[,bash,role="no-placeholders"] +---- +curl "https://api.redpanda.com/v1/clusters/" \ + -H "Authorization: Bearer " +---- + +Serverless:: ++ +Make a request to the link:/api/doc/cloud-controlplane/operation/operation-serverlessclusterservice_getserverlesscluster[`GET /v1/serverless/clusters/\{id}`] endpoint: + [,bash] ---- -rpk acl user create mcp --password +curl "https://api.redpanda.com/v1/serverless/clusters/" \ + -H "Authorization: Bearer " +---- +==== ++ +The response includes a `dataplane_api.url` value: ++ +[.no-copy] +---- + "id": "....", + "name": "my-cluster", +.... + "dataplane_api": { + "url": "https://api-xyz.abc.fmc.ppd.cloud.redpanda.com" + }, +... ---- + -Save the password securely. You need it later when configuring the MCP server. +[NOTE] +==== +The `dataplane_api.url` field might not be immediately available when a cluster reaches STATE_READY. If the field is missing or null, wait a few minutes and make the request again. The Data Plane API URL is typically available within 5-10 minutes after cluster creation completes. +==== -. Grant the `mcp` user permissions to produce and consume from the `events` topic: +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-topicservice_createtopic[`POST /v1/topics`] to create the topic: + [,bash] ---- -rpk acl create --allow-principal User:mcp --operation all --topic events +curl -X POST "https:///v1/topics" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "topic": { + "name": "events", + "partition_count": 3, + "replication_factor": 3 + } + }' ---- +-- +====== -== Create a Remote MCP Server in Redpanda Cloud +== Create an MCP Server in Redpanda Cloud +[tabs] +===== +Cloud UI:: ++ +-- . Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. . Navigate to *Remote MCP*. + This page shows a list of existing servers. -. Click *Create new MCP Server*. In *Metadata* add: +. Click *Create new MCP Server*. In *Server Metadata*, configure the basic information and resources: + * *Display Name*: A human-friendly name such as `event-data-generator`. This name is shown in the Redpanda Cloud Console. It is not the name of the MCP server itself. * *Description*: Explain what the server does. For example, `Generates fake user event data and publishes it to Redpanda topics`. -* *Tags*: Add key/value tags such as `owner=platform` or `env=demo`. +* *Tags*: Add key/value tags such as `owner=platform` or `env=demo`. The tag names `service_account_id` and `secret_id` are reserved and cannot be used. * *Resources*: Choose a size (XSmall / Small / Medium / Large / XLarge). Larger sizes allow more concurrent requests and faster processing, but cost more. You can change this later. +* *Service Account*: A service account is automatically created for authenticating the MCP server to your cluster. The name is pre-filled. You can customize this name or keep the default. ++ +[NOTE] +==== +Service accounts authenticate MCP server requests to your Redpanda cluster. The service account has editor permissions to the cluster, allowing it to perform operations like reading and writing data, managing topics, and accessing cluster resources. Service account credentials are stored in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with the ID `SERVICE_ACCOUNT_` and scope set to *MCP server*. +==== . Click *Next* to define tools. - -=== Add tools - ++ Tools define the actions your MCP server can perform. In this example, you create two tools: one for generating user event data and another for publishing that data to Redpanda. . From the *Template* dropdown, select *Generate Input*. @@ -79,9 +169,7 @@ The template populates the configuration with YAML for the tool definition. . From the *Template* dropdown, select *Redpanda Output*. + -The template populates the configuration for publishing to Redpanda and a section for adding the required secrets is displayed. - -. Enter the values for the `mcp` user's credentials in the *Add Required Secrets* section. +The template populates the configuration for publishing to Redpanda. Authentication is handled automatically by the MCP server's service account. . Click *Lint* to check the configuration. You should see no errors. @@ -93,12 +181,88 @@ It may take a few seconds to start. The status changes from *Starting* to *Runni + * For the `generate_input` tool, click *Run Tool* to generate sample event data. * For the `redpanda_output` tool, enter some sample event data such as `user_id=user123`, `event_type=login`, and `timestamp=2025-10-21T10:30:00Z` then click *Run Tool* to publish it to the `events` topic. +-- + +Data Plane API:: ++ +-- +. Create a service account using the link:/api/doc/cloud-controlplane/operation/operation-serviceaccountservice_createserviceaccount[Control Plane API]: ++ +[,bash] +---- +curl -X POST "https://api.redpanda.com/v1/service-accounts" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "service_account": { + "name": "mcp-server-demo", + "description": "Service account for MCP server demo" + } + }' +---- ++ +The response includes `client_id` and `client_secret`. Save these values. + +. Store the service account credentials in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]: ++ +* Create secret `REDPANDA_SA_CLIENT_ID` with the service account `client_id`. +* Create secret `REDPANDA_SA_CLIENT_SECRET` with the service account `client_secret`. + +. Using the Data Plane API URL from the previous section, make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_createmcpserver[`POST /v1/redpanda-connect/mcp-servers`] to create the MCP server: ++ +[,bash] +---- +curl -X POST "https:///v1/redpanda-connect/mcp-servers" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "mcp_server": { + "display_name": "event-data-generator", + "description": "Generates fake user event data and publishes it to Redpanda topics", + "tags": { + "owner": "platform", + "env": "demo" + }, + "resources": { + "memory_shares": "400M", + "cpu_shares": "100m" + }, + "service_account": { + "client_id": "${secrets.REDPANDA_SA_CLIENT_ID}", + "client_secret": "${secrets.REDPANDA_SA_CLIENT_SECRET}" + }, + "tools": { + "generate_input": { + "component_type": "COMPONENT_TYPE_INPUT", + "config_yaml": "generate:\n interval: 1s\n mapping: |\n root.user_id = \"user\" + random_int(min: 1, max: 1000).string()\n root.event_type = [\"login\", \"logout\", \"purchase\", \"view\"].index(random_int(max: 3))\n root.timestamp = now().ts_format(\"2006-01-02T15:04:05Z07:00\")" + }, + "redpanda_output": { + "component_type": "COMPONENT_TYPE_OUTPUT", + "config_yaml": "redpanda:\n seed_brokers: [ \"${REDPANDA_BROKERS}\" ]\n topic: events\n tls:\n enabled: true\n" + } + } + } + }' +---- ++ +The response includes the MCP server ID. Wait for the status to show *Running* before testing the tools. + +. To test the tools, use the link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_getmcpserver[`GET /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`] endpoint to verify the server is running. +-- +===== + +== Connect an AI client -== Connect a client +Now that your MCP server is running with two tools available, you'll connect Claude Code so it can discover and use them. You can connect any MCP-compatible AI client to your MCP server. -You can connect any MCP-compatible client to your Remote MCP server. This example uses Claude Code. +When you connect Claude Code: -The Redpanda CLI acts as a local proxy, forwarding requests from your AI client to your own Remote MCP server running in the Redpanda Cloud cluster. +. Claude automatically discovers your `generate_input` and `redpanda_output` tools +. You can ask Claude in natural language to perform tasks using these tools +. Claude decides which tools to call and in what order based on your request +. The Redpanda CLI acts as a secure proxy, forwarding Claude's tool requests to your MCP server in the cloud + +This example uses Claude Code, but the same pattern works with any MCP-compatible client. . Log in to your Redpanda Cloud account: + @@ -138,20 +302,35 @@ rpk cloud mcp proxy \ claude ---- -. Ask Claude Code to use your tools. For example: +. Ask Claude Code to use your tools. Try these example requests: + * "Generate 10 user events and then publish them to the events topic." * "Create sample login events for users user001, user002, and user003, then publish them to Redpanda." * "Generate purchase events with metadata and publish them to the events topic." ++ +Watch what happens: ++ +-- +* Claude analyzes your natural language request +* Claude identifies which tools to use (`generate_input` to create data, `redpanda_output` to publish) +* Claude calls your tools via the MCP server running in Redpanda Cloud +* You see the tool execution results in your Claude Code session +-- ++ +You may need to respond to prompts to grant Claude permission to call the tools. -You should see Claude Code calling your MCP server tools to generate and publish event data. You may need to respond to prompts for permissions to call the tools. - -When complete, you can verify the events were published by consuming from the `events` topic: - +. Verify the events were published by consuming from the `events` topic: ++ [,bash] ---- rpk topic consume events --num 10 ---- ++ +You should see the generated event data in JSON format, confirming that Claude successfully used your custom tools to generate data and publish it to Redpanda. + +== Summary + +You extended Claude's capabilities with custom tools that interact with your Redpanda cluster. Claude can now generate and publish event data through natural language commands without manual scripting. This same pattern works for any workflow: reading from topics, transforming data, querying databases, or calling external APIs. == Troubleshoot @@ -161,7 +340,6 @@ If your server is not starting or shows an error state: - Check the *Logs* tab for specific error messages. - Verify your YAML syntax by clicking *Lint*. -- Verify that the `mcp` user exists and has the correct permissions. === Connection issues @@ -181,14 +359,16 @@ If the publish tool returns errors: ```bash rpk topic list ``` -- Check that the `mcp` user has the correct ACLs: -+ -```bash -rpk acl list -``` - Use the MCP Inspector in Redpanda Cloud to test with different event data. - Check the MCP server logs for authentication or authorization errors. == Next steps -Write your own tools to extend the MCP server functionality. For more information, see: xref:ai-agents:mcp/remote/developer-guide.adoc[] +Now that you understand how MCP works, you can create custom tools for your specific use cases: + +* **Data ingestion**: Build tools that read from external APIs, databases, or file systems and publish to Redpanda topics. +* **Data processing**: Create tools that consume from topics, transform data, and write results back to Redpanda. +* **Monitoring and observability**: Add tools that query metrics, check system health, or analyze streaming data. +* **Integration workflows**: Connect Redpanda to other systems through custom tools that Claude can orchestrate. + +See xref:ai-agents:mcp/remote/developer-guide.adoc[] to learn how to build custom tools using Redpanda Connect components. diff --git a/modules/ai-agents/pages/mcp/remote/scale-resources.adoc b/modules/ai-agents/pages/mcp/remote/scale-resources.adoc new file mode 100644 index 000000000..bb63ee827 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/scale-resources.adoc @@ -0,0 +1,62 @@ += Scale Remote MCP Server Resources +:description: Learn how to scale MCP server resources up or down to match workload demands and optimize costs. + +You can scale your MCP server's resources up or down to match your workload needs. Resource allocation affects your xref:billing:billing.adoc#remote-mcp-billing-metrics[billing costs], which are charged per compute unit hour. + +== Prerequisites + +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +== Scale resources + +[tabs] +===== +Cloud UI:: ++ +-- +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the MCP server you want to scale and click its name. +. Click *Edit configuration*. +. Under *Resources*, select a new size: ++ +* *XSmall*: Lowest cost, suitable for development or light workloads +* *Small*: Light production workloads +* *Medium*: Standard production workloads +* *Large*: High-throughput workloads +* *XLarge*: Highest performance for demanding workloads + +. Click *Save* to apply the new resource allocation. ++ +The specified resources are available immediately. +-- + +Data Plane API:: ++ +-- +. link:/api/doc/cloud-dataplane/topic/topic-quickstart[Authenticate and get the base URL] for the Data Plane API. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_getmcpserver[`GET /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`] to retrieve the current configuration. +. Make a request to link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_updatemcpserver[`PATCH /v1/redpanda-connect/mcp-servers/\{mcp_server_id}`] to update the resources: ++ +[,bash] +---- +curl -X PATCH "https:///v1/redpanda-connect/mcp-servers/?update_mask=resources" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "mcp_server": { + "resources": { + "memory_shares": "2Gi", + "cpu_shares": "1000m" + } + } + }' +---- ++ +The updated resources are available immediately. +-- +===== + +[TIP] +==== +Monitor your MCP server's performance and adjust resources as needed. You can scale up during peak usage periods and scale down during quieter times to optimize costs. For compute unit definitions and pricing, see xref:billing:billing.adoc#remote-mcp-billing-metrics[MCP billing metrics]. +==== diff --git a/modules/billing/pages/billing.adoc b/modules/billing/pages/billing.adoc index c57797379..f51b7b39d 100644 --- a/modules/billing/pages/billing.adoc +++ b/modules/billing/pages/billing.adoc @@ -112,17 +112,35 @@ Replication to object storage is implemented with glossterm:Tiered Storage[]. Al == Redpanda Connect billing metrics -Pricing per pipeline depends on: +Pricing per pipeline depends on the compute units you allocate. The cost of a compute unit can vary based on the cloud provider and region you select for your cluster. -- The number of xref:develop:connect/configuration/resource-management.adoc[compute units] you allocate. -- The cost of a compute unit, which can vary based on the cloud provider and region you select for your cluster. +[cols="1,3",options="header"] +|=== +| Metric | Description + +| Compute | Tracks the server resources (vCPU and memory) a pipeline uses in compute units per hour. Where: + +1 compute unit = 0.1 CPU + 400 MB memory +|=== + +== Remote MCP billing metrics + +Remote MCP usage appears as a separate line item on your invoice and uses the same pricing structure as Redpanda Connect. + +Pricing per MCP server depends on the compute units you allocate. The cost of a compute unit can vary based on the cloud provider and region you select for your cluster. [cols="1,3",options="header"] |=== | Metric | Description -| Compute | Tracks the server resources (vCPU and memory) a pipeline uses in compute units per hour. -|=== +| Compute | Tracks the server resources (vCPU and memory) an MCP server uses in compute units per hour. Where: + +1 compute unit = 0.1 CPU + 400 MB memory +|=== + +NOTE: Compute units for Remote MCP use the same definition and rates as those for Redpanda Connect. + +MCP servers automatically emit OpenTelemetry traces to the xref:ai-agents:mcp/remote/monitor-activity.adoc#opentelemetry-traces-topic[`redpanda.otel_traces` topic]. For Serverless clusters, usage of this system-managed traces topic is not billed. You will not incur ingress, egress, storage, or partition charges for trace data. For Dedicated and BYOC clusters, standard billing metrics apply to the traces topic. == Support plans diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index 91a1b0d2d..618112571 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -8,6 +8,10 @@ This page lists new features added to Redpanda Cloud. == December 2025 +=== Remote MCP: GA + +You can now deploy managed MCP servers directly inside your Redpanda Cloud cluster with xref:ai-agents:mcp/remote/overview.adoc[Remote MCP]. Remote MCP servers give AI assistants streaming data capabilities, enabling use cases like real-time data generation, stream processing, and event publishing. + === Shadowing Redpanda Cloud now supports xref:manage:disaster-recovery/shadowing/overview.adoc[Shadowing], a disaster recovery solution that provides asynchronous, offset-preserving replication between distinct Redpanda clusters. Shadowing enables cross-region data protection by replicating topic data, configurations, consumer group offsets, ACLs, and Schema Registry data with byte-level fidelity.The shadow cluster operates in read-only mode while continuously receiving updates from the source cluster. During a disaster, you can failover individual topics or an entire shadow link to make resources fully writable for production traffic.