From efcb5d4a65edb8dc87ccd92a0a1a8df068f6aedf Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 16 Sep 2025 15:10:59 +0100 Subject: [PATCH 01/30] Add docs for remote MCP servers --- modules/ROOT/nav.adoc | 17 +- modules/ai-agents/pages/index.adoc | 15 + .../pages/mcp/local/configuration.adoc | 8 +- modules/ai-agents/pages/mcp/local/index.adoc | 5 +- .../ai-agents/pages/mcp/local/overview.adoc | 18 +- .../ai-agents/pages/mcp/local/quickstart.adoc | 10 +- modules/ai-agents/pages/mcp/overview.adoc | 51 + .../pages/mcp/remote/admin-guide.adoc | 57 ++ .../pages/mcp/remote/developer-guide.adoc | 919 ++++++++++++++++++ .../ai-agents/pages/mcp/remote/overview.adoc | 79 ++ .../pages/mcp/remote/quickstart.adoc | 197 ++++ modules/ai-agents/partials/beta.adoc | 1 + .../rpk/rpk-cloud/rpk-cloud-mcp-install.adoc | 1 + .../rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc | 4 + .../rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc | 1 + .../pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc | 4 + 16 files changed, 1363 insertions(+), 24 deletions(-) create mode 100644 modules/ai-agents/pages/index.adoc create mode 100644 modules/ai-agents/pages/mcp/overview.adoc create mode 100644 modules/ai-agents/pages/mcp/remote/admin-guide.adoc create mode 100644 modules/ai-agents/pages/mcp/remote/developer-guide.adoc create mode 100644 modules/ai-agents/pages/mcp/remote/overview.adoc create mode 100644 modules/ai-agents/pages/mcp/remote/quickstart.adoc create mode 100644 modules/ai-agents/partials/beta.adoc create mode 100644 modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc create mode 100644 modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 93899a6c8..36a1432d6 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -69,10 +69,16 @@ ** xref:security:secrets.adoc[Secrets] ** xref:security:cloud-safety-reliability.adoc[Safety and Reliability] -* xref:ai-agents:mcp/local/index.adoc[] -** xref:ai-agents:mcp/local/overview.adoc[Overview] -** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] -** xref:ai-agents:mcp/local/configuration.adoc[Configure] +* xref:ai-agents:index.adoc[AI Agents] +** xref:ai-agents:mcp/overview.adoc[MCP Overview] +** xref:ai-agents:mcp/local/index.adoc[Local MCP server for Redpanda Cloud] +*** xref:ai-agents:mcp/local/overview.adoc[Overview] +*** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] +*** xref:ai-agents:mcp/local/configuration.adoc[Configure] +** xref:ai-agents:mcp/remote/overview.adoc[Remote MCP] +*** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] +*** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] +*** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] * xref:develop:connect/about.adoc[Redpanda Connect] ** xref:develop:connect/connect-quickstart.adoc[Quickstart] @@ -487,8 +493,9 @@ ***** xref:reference:rpk/rpk-cloud/rpk-cloud-cluster-select.adoc[] **** xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[] **** xref:reference:rpk/rpk-cloud/rpk-cloud-logout.adoc[] -**** rpk cloud mcp +**** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp.adoc[] ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[] +***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[] ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[] *** xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[] **** xref:reference:rpk/rpk-cluster/rpk-cluster-config.adoc[] diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc new file mode 100644 index 000000000..3f2a55af6 --- /dev/null +++ b/modules/ai-agents/pages/index.adoc @@ -0,0 +1,15 @@ += AI Agents in Redpanda Cloud +:page-beta: true +:description: Learn about AI agents and the tools Redpanda Cloud provides for building them. +:page-layout: index +:page-aliases: develop:agents/about.adoc, develop:ai-agents/about.adoc + +include::ai-agents:partial$beta.adoc[] + +AI agents are configurable digital assistants that autonomously perform specialist tasks by leveraging large language models (LLMs) and connecting to external data sources and tools. + +Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: + +**Local MCP server for Redpanda Cloud**: A built-in server that gives your AI assistant direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. + +**Remote MCP (managed)**: Your own custom MCP servers built with Redpanda Connect pipelines and hosted inside your Redpanda Cloud cluster. These let you create reusable tools for your team that integrate with knowledge bases, APIs, and web content while running close to your data. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/configuration.adoc b/modules/ai-agents/pages/mcp/local/configuration.adoc index 60781047d..22861fcb2 100644 --- a/modules/ai-agents/pages/mcp/local/configuration.adoc +++ b/modules/ai-agents/pages/mcp/local/configuration.adoc @@ -1,8 +1,8 @@ -= Configure Redpanda Cloud MCP += Configure the Local MCP server for Redpanda Cloud :page-beta: true -:description: Learn how to configure the Redpanda Cloud MCP Server for your AI assistant, including auto and manual client setup, enabling deletes, and security considerations. +:description: Learn how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. -This page explains how to configure the Redpanda Cloud MCP Server for your AI assistant, including auto and manual client setup, enabling deletes, and security considerations. +This page explains how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. == Prerequisites @@ -31,7 +31,7 @@ If you need to update the integration, re-run the install command for your clien == Manual configuration for other MCP clients -If you're using another MCP-compatible client, you can manually configure it to use Redpanda Cloud MCP Server. Follow these steps: +If you're using another MCP-compatible client, you can manually configure it to use the local MCP server for Redpanda Cloud. Follow these steps: Add an MCP server entry to your client's configuration (example shown in JSON). Adjust paths for your system. diff --git a/modules/ai-agents/pages/mcp/local/index.adoc b/modules/ai-agents/pages/mcp/local/index.adoc index e70b573f8..16429590c 100644 --- a/modules/ai-agents/pages/mcp/local/index.adoc +++ b/modules/ai-agents/pages/mcp/local/index.adoc @@ -1,5 +1,4 @@ -= Redpanda Cloud MCP Server += Local MCP server for Redpanda Cloud :page-beta: true -:description: Find links to information about the Redpanda Cloud MCP Server and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. +:description: Find links to information about the Local MCP server for Redpanda Cloud and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. :page-layout: index -:page-aliases: develop:agents/about.adoc \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/overview.adoc b/modules/ai-agents/pages/mcp/local/overview.adoc index 8160dccc3..cc624f400 100644 --- a/modules/ai-agents/pages/mcp/local/overview.adoc +++ b/modules/ai-agents/pages/mcp/local/overview.adoc @@ -1,16 +1,16 @@ -= About the Redpanda Cloud MCP Server += About the Local MCP server for Redpanda Cloud :page-beta: true -:description: Learn about the Redpanda Cloud MCP Server, which lets AI assistants securely access and operate your Redpanda Cloud account and clusters. +:description: Learn about the local MCP server for Redpanda Cloud, which lets AI assistants securely access and operate your Redpanda Cloud account and clusters. -The Redpanda Cloud MCP Server lets AI assistants securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: +The local MCP server for Redpanda Cloud lets AI assistants securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: * link:https://docs.redpanda.com/api/doc/cloud-controlplane/[Control Plane] actions, such as creating a Redpanda Cloud cluster or listing clusters. * link:https://docs.redpanda.com/api/doc/cloud-dataplane/[Data Plane] actions, such as creating topics or listing topics. By speaking natural language to your assistant, you can ask it to perform Redpanda operations on your behalf. The MCP server runs locally on your machine and authenticates to Redpanda Cloud using a Redpanda Cloud token. -image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking Redpanda Cloud MCP to list topics in a cluster.] +image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking the local MCP server for Redpanda Cloud to list topics in a cluster.] == What you can do @@ -32,7 +32,7 @@ NOTE: The MCP server does **not** expose delete endpoints by default. You can en . Authenticate to Redpanda Cloud and receive a token using the xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command]. . Configure your MCP client using the xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[`rpk cloud mcp install`] command. Your client then starts the server on-demand using xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[`rpk cloud mcp stdio`], authenticating with the Redpanda Cloud token from `rpk cloud login`. -. Prompt your assistant to perform Redpanda operations. The Redpanda Cloud MCP Server executes them in your Redpanda Cloud account using your Redpanda Cloud token. +. Prompt your assistant to perform Redpanda operations. The local MCP server for Redpanda Cloud executes them in your Redpanda Cloud account using your Redpanda Cloud token. === Components @@ -40,6 +40,14 @@ NOTE: The MCP server does **not** expose delete endpoints by default. You can en * Redpanda CLI (`rpk`) for obtaining a token and starting the local MCP server. * Redpanda Cloud account that the local MCP server can connect to and issue API requests. +== Security considerations + +MCP servers authenticate to Redpanda Cloud using your personal or service account credentials. However, there is **no auditing or access control** that distinguishes between actions performed by MCP servers versus direct API calls: + +* All API actions appear in Redpanda Cloud's internal logs as coming from the authenticated user account, not the specific MCP server. +* You cannot audit which MCP server performed which operations, as Redpanda Cloud logs are not accessible to users. +* You cannot restrict specific MCP servers to only certain API endpoints or resources. + == Next steps * xref:ai-agents:mcp/local/quickstart.adoc[] diff --git a/modules/ai-agents/pages/mcp/local/quickstart.adoc b/modules/ai-agents/pages/mcp/local/quickstart.adoc index 55eddc00b..6cf7ee111 100644 --- a/modules/ai-agents/pages/mcp/local/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/local/quickstart.adoc @@ -1,14 +1,10 @@ -= Redpanda Cloud MCP Quickstart += Local MCP server for Redpanda Cloud Quickstart :page-beta: true -:description: Connect your Claude AI assistant to your Redpanda Cloud account and clusters using the Redpanda Cloud MCP Server. +:description: Connect your Claude AI assistant to your Redpanda Cloud account and clusters using the local MCP server for Redpanda Cloud. -In this quickstart, you'll get your Claude AI assistant talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[Redpanda Cloud MCP Server]. - -//// -To be used when Remote MCP is ready for public beta +In this quickstart, you'll get your Claude AI assistant talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud]. If you're trying to deploy your own MCP server as a managed service inside your cluster, see xref:ai-agents:mcp/remote/quickstart.adoc[]. -//// == Prerequisites diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc new file mode 100644 index 000000000..826e3c3df --- /dev/null +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -0,0 +1,51 @@ += MCP in Redpanda Cloud +:page-beta: true +:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Local MCP server for Redpanda Cloud and remote MCP (managed). + +include::ai-agents:partial$beta.adoc[] + +**Model Context Protocol (MCP)** is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. + +== Why use MCP with Redpanda Cloud? + +With MCP, you can have natural conversations with your AI assistant about your streaming data infrastructure. Rather than remembering command-line syntax or navigating through web interfaces, you describe what you want in plain English and your AI assistant handles the technical details. + +== MCP options in Redpanda Cloud + +In Redpanda Cloud, there are two complementary options to use MCP: + +[cols="1s,2a,2a"] +|=== +| | Local MCP server for Redpanda Cloud | Remote MCP (managed) + +| Description +| A server that gives your AI assistant access to your Redpanda Cloud account and clusters. +| Your own MCP server built with Redpanda Connect and hosted inside your Redpanda Cloud cluster. + +| Where it runs +| Your computer +| In your Redpanda Cloud cluster (managed by Redpanda) + +| Who builds it +| Redpanda +| You, with Redpanda Connect + +| Example use cases (prompts you might ask) +| - Create a new cluster called `dev-analytics` for testing. +- List all topics in my production cluster. +| - Analyze the last 100 orders and show me the top product categories. +- Generate a summary of user activity from the events stream. +|=== + +TIP: The Redpanda documentation site also has a read-only MCP server that provides access to Redpanda docs and examples. This server has no access to your Redpanda Cloud account or clusters. See xref:home:ROOT:mcp-setup.adoc[]. + +== Which should I use? + +Use the **local MCP server for Redpanda Cloud** to operate your Redpanda Cloud account quickly from your laptop. + +Use the **remote MCP** to provide reusable tools for your team, running inside the cluster, authored with Redpanda Connect. + +== Get started + +* xref:ai-agents:mcp/local/quickstart.adoc[] +* xref:ai-agents:mcp/remote/quickstart.adoc[] diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc new file mode 100644 index 000000000..de5821675 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -0,0 +1,57 @@ += Manage Remote MCP Servers +:page-beta: true +:description: Learn how to manage, edit, stop, resume, and delete remote MCP servers in Redpanda Cloud. + +include::ai-agents:partial$beta.adoc[] + +This guide is for Redpanda Cloud administrators who need to manage the lifecycle of remote MCP servers. + +== Prerequisites + +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +== Edit an existing MCP server + +You can update the configuration, resources, or metadata of a remote MCP server at any time. + +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the MCP server you want to edit and click its name. +. Click *Edit configuration*. +. Make your changes. +. Click *Save* to apply changes. + +[NOTE] +==== +Editing a running MCP server may cause a brief interruption. Review changes before deploying to production. +==== + +== Stop (pause) an MCP server + +Pausing a server stops all tool execution and releases compute resources, but preserves configuration and state. + +. In the *Remote MCP* list, find the server you want to stop. +. Click the three dots and select *Stop*. +. Confirm the action. + +[NOTE] +==== +While stopped, the server does not respond to MCP requests. Start it again to restore service. +==== + +== Resume a stopped MCP server + +. In the *Remote MCP* list, find the stopped server. +. Click the three dots and select *Start*. +. Wait for the status to show *Running* before reconnecting clients. + +== Delete an MCP server + +Deleting a server is permanent and cannot be undone. + +. In the *Remote MCP* list, find the server to delete. +. Click the three dots and select *Delete*. +. Confirm the deletion when prompted. + +== API and automation + +All of these actions can also be performed using the Redpanda Cloud API version v1alpha3. See the link:https://github.com/redpanda-data/console/blob/master/proto/gen/openapi/openapi.v1alpha3.yaml[OpenAPI specification file] for endpoints to manage MCP servers. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc new file mode 100644 index 000000000..8f7e09773 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -0,0 +1,919 @@ += Build Remote MCP Servers in Redpanda Cloud +:page-beta: true +:description: Learn how to write and deploy remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. + +include::ai-agents:partial$beta.adoc[] + +This guide provides a reference for building remote MCP servers that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. + +== Prerequisites + +* Access to a Redpanda Cloud cluster +* Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries +* (Optional) Claude / Claude Code for testing + +TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +You should also be familiar with YAML, HTTP APIs, and basic event-stream processing concepts. + +== Concepts and architecture + +Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write tool logic as Redpanda Connect pipelines and annotate them with MCP metadata so clients can discover and invoke them. + +Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your remote MCP servers are implemented as Redpanda Connect pipelines that run inside your Redpanda Cloud cluster. + +Tool:: A single request/response operation exposed to MCP. Each tool is implemented as a Redpanda Connect pipeline and described to MCP with `meta.mcp`. + +Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use *UPPER_CASE snake_case* for secret names. + +== Development workflow + +. <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. +. <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. +. <>: Build an MCP tool using Redpanda Connect pipelines. See <> for examples of what is allowed in a pipeline. +. <>: Validate your pipeline, catch schema errors, then deploy and test with the MCP Inspector. +. <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. +. <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. + +[[contract]] +== Tool contract and MCP metadata + +Each MCP tool must declare its interface using `meta.mcp` metadata. This metadata allows AI clients to discover and invoke the tool correctly. + +Define a clear, stable interface for each tool. Keep the description task-oriented and keep parameters to a minimum. + +[source,yaml] +---- +meta: + mcp: + enabled: true + description: "Fetches a compact summary from an external API using two optional parameters." + properties: + - name: parameter1 + type: string + description: "Primary filter; defaults to provider standard when omitted." + required: false + - name: parameter2 + type: number + description: "Limit of results (1-100)." + required: false +---- + +Property guidance: + +* Use `string`, `number`, or `boolean` types. +* Validate ranges and enums using Bloblang inside the pipeline. +* Mark only mandatory fields as required. +* Document defaults in the `description` and enforce them in the pipeline. + +[[provision-secrets]] +== Provision secrets + +All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Follow these best practices: + +- Reference secrets as `${secrets.NAME}` in your pipeline YAML. +- Never commit secrets to Git or reference them directly inline in configuration files. +- Use `UPPER_CASE` snake_case for secret names (such as `DATAPLANE_TOKEN`). +- Rotate secrets in the Secrets Store as needed. +- Only request the scopes/roles required by your tool (principle of least privilege). + +See an example of using secrets in the <> section. + +[[pipeline-patterns]] +== Pipeline patterns + +Remote MCP tools are implemented as Redpanda Connect pipelines. This section shows you how to build different types of MCP tools using various pipeline patterns, organized from basic to advanced. + +TIP: For detailed configuration options for any component, see the xref:develop:connect/components/about.adoc[Redpanda Connect components reference]. + +=== Basic patterns + +These patterns cover the most common MCP tool use cases. Start with these if you're new to building remote MCP tools. + +==== Data generators (input patterns) + +Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. + +This example generates a realistic user event message: + +[source,yaml] +---- +label: generate_input +generate: + mapping: | + let event_type = ["login", "logout", "purchase", "view_page", "click_button"].index(random_int(max:4)) + root = { + "id": uuid_v4(), + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "user_id": random_int(min:1, max:10000), + "event_type": $event_type, + "data": { + "session_id": ksuid(), + "ip_address": "192.168.%v.%v".format(random_int(max:255), random_int(min:1, max:254)), + "user_agent": ["Chrome", "Firefox", "Safari", "Edge"].index(random_int(max:3)), + "amount": if $event_type == "purchase" { random_int(min:10, max:500) } else { null } + } + } + +meta: + mcp: + enabled: true + description: "Generate an example user event message with realistic data" + properties: [] +---- + + } + +See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] + +==== External API calls (processor patterns) + +Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. + +[source,yaml] +---- +label: http_processor +processors: + - label: prepare_parameters + mutation: | + meta city_name = this.city_name + - label: fetch_weather + http: + url: 'https://wttr.in/${! @city_name }?format=j1' + verb: GET + headers: + Accept: "application/json" + User-Agent: "redpanda-mcp-server/1.0" + - label: format_response + mutation: | + root = { + "city": @city_name, + "temperature": this.current_condition.0.temp_C.number(), + "feels_like": this.current_condition.0.FeelsLikeC.number(), + "humidity": this.current_condition.0.humidity.number(), + "pressure": this.current_condition.0.pressure.number(), + "description": this.current_condition.0.weatherDesc.0.value, + "wind_speed": this.current_condition.0.windspeedKmph.number(), + "metadata": { + "source": "wttr.in", + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Fetch current weather information for a specified city" + properties: + - name: city_name + type: string + description: "Name of the city to get weather information for" + required: true +---- + +See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] + +==== Database queries (processor patterns) + +Query external databases and return structured results. This pattern is essential for tools that need to access business data. + +[source,yaml] +---- +label: gcp_bigquery_select_processor +processors: + - label: prepare_parameters + mutation: | + meta customer_id = this.customer_id.string().catch("12345") + meta limit = this.limit.number().catch(10) + - label: query_bigquery + gcp_bigquery_select: + project: my-gcp-project + credentials_json: | + ${secrets.BIGQUERY_CREDENTIALS} + table: my_dataset.customer_orders + columns: + - "order_id" + - "customer_id" + - "order_date" + - "total_amount" + - "status" + where: customer_id = ? AND order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) + suffix: "ORDER BY order_date DESC LIMIT ?" + args_mapping: root = [ @customer_id, @limit ] + - label: format_response + mutation: | + root = { + "orders": this, + "metadata": { + "source": "BigQuery", + "customer_id": @customer_id, + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Query customer orders from BigQuery" + properties: + - name: customer_id + type: string + description: "Customer ID to filter orders" + required: true + - name: limit + type: number + description: "Maximum number of orders to return" + required: false +---- + +See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) + +=== Intermediate patterns + +These patterns handle more complex scenarios like data publishing, caching, and topic inspection. Use these when you need to integrate with Redpanda infrastructure or handle stateful operations. + +==== Data publishers (output patterns) + +Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems or build caching systems. + +This example publishes a message to a Redpanda topic: + +[source,yaml] +---- +label: redpanda_output +redpanda: + seed_brokers: + - ${REDPANDA_BROKERS} + topic: ${! this.topic_name.string().catch("default-topic") } + timeout: 30s + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: ${secrets.REDPANDA_USERNAME} + password: ${secrets.REDPANDA_PASSWORD} +meta: + mcp: + enabled: true + description: Publishes a message to a specified Redpanda topic + properties: + - name: message + type: string + description: The message content to publish + required: true + - name: topic_name + type: string + description: The Redpanda topic to publish to + required: true +---- + +See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +==== Caching systems (cache patterns) + +Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. + +[source,yaml] +---- +label: redpanda_cache +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] + topic: "mcp-cache-topic" + tls: + enabled: true + sasl: + - mechanism: "SCRAM-SHA-512" + username: "${secrets.MCP_REDPANDA_CREDENTIALS.username}" + password: "${secrets.MCP_REDPANDA_CREDENTIALS.password}" + +meta: + mcp: + enabled: true + description: "Redpanda-backed distributed cache using Kafka topics for persistence" +---- + +This example implements an in-memory cache for low-latency access to small datasets: + +[source,yaml] +---- +label: memory_cache +memory: + default_ttl: "5m" + init_values: + "user:1001": '{"name": "Alice", "role": "admin"}' + "user:1002": '{"name": "Bob", "role": "user"}' + "config:theme": "dark" + "config:language": "en" + shards: 4 + +meta: + mcp: + enabled: true + description: "In-memory cache for storing user data, configuration, and temporary values" +---- + +See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +=== Advanced patterns + +These patterns handle complex scenarios like multi-step workflows, production observability, and sophisticated error handling. Use these when building enterprise-grade tools. + +==== Parameter validation and type coercion + +Always validate and coerce input parameters to ensure your tools are robust: + +[source,yaml] +---- +processors: + - label: validate_params + mutation: | + # Validate required parameters + root = if !this.exists("user_id") { + throw("user_id parameter is required") + } else { this } + + # Type coercion with validation + meta user_id = this.user_id.string() + meta limit = this.limit.number().catch(10) + meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) + + # Range validation + meta validated_limit = if @limit < 1 || @limit > 1000 { + throw("limit must be between 1 and 1000") + } else { @limit } +---- + +==== Dynamic configuration + +Build tools that adapt their behavior based on input parameters: + +[source,yaml] +---- +processors: + - label: dynamic_config + mutation: | + # Choose data source based on environment + meta env = this.environment | "production" + meta table_name = match @env { + "dev" => "dev_orders", + "staging" => "staging_orders", + "production" => "prod_orders", + _ => "dev_orders" + } + + # Adjust query complexity based on urgency + meta columns = if this.detailed.bool().catch(false) { + ["order_id", "customer_id", "total", "items", "shipping_address"] + } else { + ["order_id", "customer_id", "total"] + } +---- + +==== Error handling and fallbacks + +Implement error handling to make your tools reliable: + +[source,yaml] +---- +processors: + - label: primary_fetch + try: + - http: + url: "https://api.primary.com/data" + timeout: "10s" + catch: + - log: + message: "Primary API failed, trying fallback" + - label: fallback_fetch + http: + url: "https://api.fallback.com/data" + timeout: "15s" + - mutation: | + root.metadata.source = "fallback" + root.metadata.warning = "Primary source unavailable" +---- + +==== Conditional processing + +Build tools that branch based on input or data characteristics: + +[source,yaml] +---- +processors: + - label: conditional_processing + switch: + - check: this.data_type == "json" + processors: + - json: + operator: "parse" + - mutation: 'root.parsed_data = this' + - check: this.data_type == "csv" + processors: + - csv: + parse: true + - mutation: 'root.parsed_data = this' + - processors: + - mutation: 'root.error = "Unsupported data type"' +---- + +[[secrets]] +==== Secrets and credentials + +Securely handle multiple credentials and API keys. + +Here is an example of using an API key secret. + +. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. + +. Reference the secret in your pipeline configuration: ++ +[source,yaml] +---- +processors: + - label: call_external_api + http: + url: "https://api.example.com/data" + verb: GET + headers: + Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> + Accept: "application/json" +---- ++ +<1> The secret is injected at runtime. Never store the actual API key in your YAML + +IMPORTANT: The `${secrets.NAME}` syntax is resolved by Redpanda Connect at runtime. The actual secret value never appears in your configuration files or logs. + +==== Monitoring, debugging, and observability + +Use structured logging, request tracing, and performance metrics to gain insights into tool execution. + +[source,yaml] +---- +label: observable_tool +processors: + - label: init_tracing + mutation: | + # Generate correlation ID for request tracing + meta req_id = uuid_v7() + meta start_time = now() + + # Log request start with structured data + root.trace = { + "request_id": @req_id, + "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "version": "1.0.0" + } + + - label: log_request_start + log: + message: "MCP tool request started" + fields: + request_id: "${! @req_id }" + tool_name: "observable_tool" + input_params: "${! this.without(\"trace\") }" + user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }" + level: "INFO" + + - label: finalize_response + mutation: | + # Calculate total execution time + meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000 + + # Add trace information to response + root.metadata = { + "request_id": @req_id, + "execution_time_ms": @duration, + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "success": !this.exists("error") + } + + - label: log_completion + log: + message: "MCP tool request completed" + fields: + request_id: "${! @req_id }" + duration_ms: "${! this.metadata.execution_time_ms }" + success: "${! this.metadata.success }" + result_size: "${! content().length() }" + level: "INFO" + +meta: + mcp: + enabled: true + description: "Example tool with comprehensive observability and error handling" + properties: + - name: user_id + type: string + description: "User ID to fetch data for" + required: true +---- + +Observability features: + +* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing +* *Execution timing*: Track how long your tools take to execute using nanosecond precision +* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` +* *Request/response metadata*: Log input parameters and response characteristics +* *Success tracking*: Monitor whether operations complete successfully + +You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: + +[source,json] +---- +{ + "metadata": { + "execution_time_ms": 0.158977, + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "success": true, + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool" + }, + "trace": { + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool", + "version": "1.0.0" + }, + "user_id": "1" +} +---- + +For more advanced observability, use Redpanda Connect's GCP Cloud Trace integration to track request flows across your MCP tools: + +[source,yaml] +---- +# Configure tracer resource for GCP Cloud Trace +tracer_resources: + - label: gcp_tracer + gcp_cloudtrace: + project: "${secrets.GCP_PROJECT_ID}" + sampling_ratio: 1.0 + tags: + environment: "production" + cluster: "${CLUSTER_ID}" + service: "mcp-tools" + flush_interval: "5s" + +label: traced_tool +processors: + - label: start_span + tracer: + resource: gcp_tracer + operation_name: "mcp_tool_execution" + tags: + tool.name: "user_data_fetcher" + tool.version: "1.2.0" + user.id: "${! this.user_id }" + + - label: fetch_user_data + http: + url: "https://api.example.com/users/${! this.user_id }" + verb: GET + headers: + Authorization: "Bearer ${secrets.API_TOKEN}" + timeout: "10s" + + - label: add_span_details + tracer: + resource: gcp_tracer + tags: + http.status_code: "${! meta(\"http_status_code\") }" + response.size: "${! content().length() }" + api.endpoint: "users" + success: "${! !error().exists() }" + +meta: + mcp: + enabled: true + description: "User data fetcher with GCP Cloud Trace integration" + properties: + - name: user_id + type: string + required: true +---- + +Tracer benefits for MCP tools: + +* *Request flow visualization*: See how requests flow through your MCP tools and external services +* *Performance bottleneck identification*: Identify slow API calls or processing steps +* *Cross-service correlation*: Trace requests that span multiple services and systems +* *Error attribution*: Quickly identify which component caused a failure +* *Dependency mapping*: Understand relationships between your MCP tools and external services + +See also: xref:develop:connect/components/tracers/gcp_cloudtrace.adoc[`gcp_cloudtrace` tracer], xref:develop:connect/components/processors/log.adoc[`log` processor], xref:develop:connect/components/processors/try.adoc[`try` processor], xref:develop:connect/guides/bloblang/functions.adoc[Bloblang functions] (for timing and ID generation) + +==== Multi-step data enrichment + +Build tools that combine data from multiple sources. + +This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. + +[source,yaml] +---- +label: customer_enrichment +processors: + - label: fetch_customer_base + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "customers" + where: "customer_id = ?" + args_mapping: 'root = [this.customer_id]' + + - label: enrich_with_orders + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "orders" + where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" + args_mapping: 'root = [this.customer_id]' + + - label: combine_data + mutation: | + root = { + "customer": this.customers.index(0), + "recent_orders": this.orders, + "metrics": { + "total_orders": this.orders.length(), + "total_spent": this.orders.map_each(o -> o.total).sum(), + "avg_order_value": this.orders.map_each(o -> o.total).mean() + } + } + +meta: + mcp: + enabled: true + description: "Get comprehensive customer profile with recent order history and metrics" + properties: + - name: customer_id + type: string + description: "Customer ID to analyze" + required: true +---- + +See also: xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) + +==== Workflow orchestration + +Coordinate complex workflows with multiple steps and conditional logic. + +This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. + +[source,yaml] +---- +label: order_workflow +processors: + - label: validate_order + mutation: | + # Validation logic + root = if this.total <= 0 { + throw("Invalid order total") + } else { this } + + - label: mock_inventory_check + mutation: | + # Mock inventory data for testing + let inventory = { + "widget-001": {"quantity": 100, "name": "Standard Widget"}, + "widget-premium": {"quantity": 25, "name": "Premium Widget"}, + "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"} + } + + let product = $inventory.get(this.product_id) + root = if $product == null { + throw("Product not found: " + this.product_id) + } else if $product.quantity < this.quantity { + throw("Insufficient inventory. Available: " + $product.quantity.string()) + } else { + this.merge({ + "inventory_check": "passed", + "available_quantity": $product.quantity, + "product_name": $product.name + }) + } + + - label: route_by_priority + switch: + - check: 'this.total > 1000' + processors: + - label: mock_high_value_processing + mutation: | + # Mock premium processing + root = this.merge({ + "processing_tier": "premium", + "processing_time_estimate": "2-4 hours", + "assigned_rep": "premium-team@company.com", + "priority_score": 95 + }) + + - check: 'this.customer_tier == "vip"' + processors: + - label: mock_vip_processing + mutation: | + # Mock VIP processing + root = this.merge({ + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "assigned_rep": "vip-team@company.com", + "priority_score": 90, + "perks": ["expedited_shipping", "white_glove_service"] + }) + + - processors: + - label: mock_standard_processing + mutation: | + # Mock standard processing + root = this.merge({ + "processing_tier": "standard", + "processing_time_estimate": "24-48 hours", + "assigned_rep": "support@company.com", + "priority_score": 50 + }) + + - label: finalize_order + mutation: | + # Add final processing metadata + # Calculate estimated fulfillment by parsing processing time + let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number() + + root = this.merge({ + "order_status": "processed", + "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"), + "estimated_fulfillment": "TBD - calculated based on processing tier", + "processing_time_hours": $max_hours + }) + +meta: + mcp: + enabled: true + description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)" + properties: + - name: order_id + type: string + description: "Unique order identifier" + required: true + - name: product_id + type: string + description: "Product ID (try: widget-001, widget-premium, widget-limited)" + required: true + - name: quantity + type: number + description: "Quantity to order" + required: true + - name: total + type: number + description: "Order total in dollars" + required: true + - name: customer_tier + type: string + description: "Customer tier (optional: vip, standard)" + required: false +---- + +For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: + +[source,json] +---- +{ + "assigned_rep": "vip-team@company.com", + "available_quantity": 100, + "customer_tier": "vip", + "estimated_fulfillment": "TBD - calculated based on processing tier", + "inventory_check": "passed", + "order_id": "ORD001", + "order_status": "processed", + "perks": [ + "expedited_shipping", + "white_glove_service" + ], + "priority_score": 90, + "processed_at": "2025-09-16T09:05:29.138Z", + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "processing_time_hours": 2, + "product_id": "widget-001", + "product_name": "Standard Widget", + "quantity": 5, + "total": 250 +} +---- + +Notice how the workflow: + +. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. +. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. +. Routes by customer tier: Since `customer_tier` is "vip", it gets VIP processing with special `perks` and priority. +. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. +. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. + +[[lint]] +== Deploy and test your MCP server + +When your pipeline is ready, deploy it as a remote MCP server in Redpanda Cloud and test it with the built-in MCP Inspector. + +=== Access MCP Servers in Redpanda Cloud + +. Log into your Redpanda Cloud Console at https://cloud.redpanda.com. +. Select your cluster from the cluster list. +. In the left sidebar, navigate to *Remote MCP*. +. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button + +=== Create and deploy your MCP server + +. Click *Create MCP Server* to open the creation form. +. Fill in the required fields: ++ +* *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). +* *Description*: Brief description of what your server does. +* *Configuration*: Paste your YAML pipeline configuration. +. Click *Lint* to check your YAML syntax and configuration. +. Click *Create MCP Server* to deploy the server. +. Monitor the *Status* column. It will show one of: ++ +* *Starting*: Server is being deployed. +* *Running*: Server is active and ready to receive requests. +* *Error*: Check logs for deployment issues. + +=== Test with MCP Inspector + +. When your server status shows *Running*, click on the server name. +. Navigate to the *MCP Inspector* tab. +. This built-in tool lets you: ++ +* View all available tools your server exposes. +* Test tool calls with sample parameters. +* See request/response data. +* Debug issues without setting up external clients. + +=== Manage your MCP server + +From the MCP Servers list, you can: + +* *View logs*: Click the server name, then go to the *Logs* tab to see execution logs and errors. +* *Update configuration*: Edit the YAML and redeploy. +* *Start/Stop*: Control server lifecycle. +* *Delete*: Remove the server and clean up resources. + +See xref:ai-agents:mcp/remote/admin-guide.adoc[]. + +[[test]] +== Authenticate and connect your AI client + +To connect your local AI client to your remote MCP server: + +. First, authenticate to Redpanda Cloud: ++ +[source,bash] +---- +rpk cloud login +---- ++ +This opens a browser window for sign-in and stores your authentication token locally. + +. Install the MCP proxy for your client (Claude/Claude Code): ++ +[source,bash] +---- +rpk cloud mcp proxy \ + --cluster-id \ <1> + --mcp-server-id \ <2> + --install --client claude-code +---- ++ +<1> `--cluster-id`: The target Redpanda Cloud cluster ID where your remote MCP server is hosted. +<2> `--mcp-server-id`: The unique ID of your deployed remote MCP server ++ +TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. ++ +The proxy acts as a bridge between your local AI client and the remote MCP server running in your cluster. It: ++ +* Connects to your remote MCP server using your authentication token +* Discovers and registers all tools from the remote server locally +* Proxies MCP requests from your AI client to the remote server +* Handles authentication by injecting your token into each request + +. Restart your client and invoke your tool. + +:note-caption: Building your own agent? + +[NOTE] +==== +You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement of remote MCP. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. +==== + +:note-caption: Note + +[[observe]] +== Observe and debug your MCP tools + +You can view execution logs in the Redpanda Cloud Console under your MCP server's *Logs* tab. + +== Suggested reading + +* xref:develop:connect/components/about.adoc[Redpanda Connect components reference] +* xref:develop:connect/guides/bloblang/about.adoc[Bloblang language guide] +* xref:develop:connect/configuration/secret-management.adoc[Secret management in Redpanda Connect] +* xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command reference] +* xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[`rpk cloud mcp proxy` command reference] diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc new file mode 100644 index 000000000..829a536b1 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -0,0 +1,79 @@ += Remote MCP Servers for Redpanda Cloud +:page-beta: true +:description: Learn about remote MCP servers, which are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. + +include::ai-agents:partial$beta.adoc[] + +Remote MCP servers are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. Unlike the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud], remote servers are deployed to your cluster and operated by Redpanda. Your server can include one or more tools that you write using Redpanda Connect pipelines. + +== Why use remote MCP? + +* *Always-on and centrally managed:* No need to run a local process. Your tools live with your data. +* *Proximity to data:* Execute tools next to your cluster for lower latency and simpler networking. +* *Secure by design:* Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in pipelines. + +== Architecture + +* *Build*: You describe tools as Redpanda Connect components with `meta.mcp` metadata. +* *Host*: Redpanda Cloud runs your server within the target cluster using the resources you select. +* *Access*: Clients such as Claude Code connect and authenticate using either the MCP proxy or direct connections. + +== Core concepts + +The remote MCP server is a managed service that runs inside your Redpanda Cloud cluster. You create and deploy the server using the Redpanda Cloud Console or the API. + +=== Redpanda Connect + +Remote MCP servers are implemented as Redpanda Connect pipelines. Your tools: + +* Use processors for request/response logic. +* Can access brokers using the `$\{REDPANDA_BROKERS}` environment variable (the pipeline is hosted within the cluster). + +=== Secrets + +* Store secrets in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] and reference them in configs as `${secrets.MY_SECRET_NAME}`. +* *Never* include secrets in plain text within pipelines or MCP server definitions. +* Secret names should be *UPPER_CASE snake_case*. + +=== Linting and validation + +Always use the *Lint* action before deploying to ensure the configuration compiles and required parameters are present. + +== Use cases + +[cols="1,3a"] +|=== +|Category |Example prompts + +|Operational monitoring +|* Check partition lag for customer-events topic +* Show me the top 10 producers by message volume today +* Get schema registry health status + +|Data enrichment and analysis +|* Fetch user profile data and recent orders for customer ID 12345 +* Get real-time stock prices for symbols in my portfolio topic +* Analyze sentiment of latest product reviews + +|Team productivity +|* Deploy my microservice to the staging environment +* Generate load test data for the payments service +* Create a summary dashboard of this week's incident reports + +|Business intelligence +|* What are the trending products in the last 24 hours? +* Show revenue impact of the latest feature deployment +* Get customer satisfaction scores from support tickets +|=== + +== How it works + +. **Authentication:** The AI client (such as Claude Code) must authenticate to Redpanda Cloud and receive a token. You can do this by using either the MCP proxy or by implementing the MCP client flow yourself. +.. *MCP proxy*: For tools with built-in MCP clients, first authenticate with `rpk cloud login`, then configure them to use the `rpk cloud mcp proxy` command. This command handles proxying requests to the remote MCP server running in your Redpanda Cloud cluster using your stored authentication token. +.. **Direct option:** If you are building your own AI agent, you can implement the MCP client flow yourself and connect directly to the remote MCP server, handling authentication and requests without `rpk`. +. **Prompting:** After connecting, you can prompt your assistant to perform operations using your remote MCP server. + +== Next steps + +* xref:ai-agents:mcp/remote/quickstart.adoc[] +* xref:ai-agents:mcp/remote/developer-guide.adoc[] diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc new file mode 100644 index 000000000..4adf883d4 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -0,0 +1,197 @@ += Remote MCP Server Quickstart +:page-beta: true +:description: Create and host a managed MCP server inside your Redpanda Cloud cluster. + +include::ai-agents:partial$beta.adoc[] + +{description} + +== Prerequisites + +* A Redpanda Cloud cluster and subscription with *Remote MCP* enabled +* Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials (if needed) +* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] +* link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed + +TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. + +== Create a remote MCP Server in Redpanda Cloud + +. Navigate to *Remote MCP*. ++ +This page shows a list of existing servers. + +. Click *Create new MCP Server*. In *Metadata* add: ++ +* *Display Name*: A human-friendly name such as `crypto-inspector`. This name is shown in the Redpanda Cloud Console. It is not the name of the MCP server itself. +* *Description*: Explain what the server does. For example, `Fetches price history of cryptocurrencies from the CryptoCompare API`. +* *Tags*: Add key/value tags such as `owner=platform` or `env=prod`. +* *Resources*: Choose a size (XSmall / Small / Medium / Large / XLarge). Larger sizes allow more concurrent requests and faster processing, but cost more. You can change this later. + +. Click *Next* to define a tool. + +. Add the name of the tool, such as `btc_price_fetcher`. This is the name clients will use to call the tool. Make it descriptive so clients can discover it. + +. Paste the following example YAML into the editor. This tool fetches Bitcoin price history from the CryptoCompare API. It accepts two optional parameters: `days` (number of days of history, default 30) and `currency` (target currency, default USD). ++ +.Reveal pipeline YAML +[%collapsible] +==== +[,yaml] +---- +label: btc_price_fetcher <1> + +processors: + - label: prepare_parameters <2> + mutation: | + meta days = this.days.catch(30) + meta currency = this.currency.catch("USD") + meta api_endpoint = "https://min-api.cryptocompare.com/data/v2/histoday?fsym=BTC&tsym=" + @currency + "&limit=" + (@days).string() + + - label: fetch_data <3> + http: + url: '${! @api_endpoint }' + verb: GET + headers: + Accept: "application/json" + User-Agent: "redpanda-mcp-server/1.0" + timeout: "30s" + retries: 3 + retry_period: "5s" + + - label: format_response <4> + mutation: | + root = if this.Response == "Success" { + { + "success": true, + "data": { + "symbol": "BTC", + "currency": @currency, + "period_days": @days.number(), + "total_records": this.Data.Data.length(), + "prices": if this.Data.Data != null { + this.Data.Data.map_each(entry -> { + "date": (entry.time.number() * 1000).ts_format("2006-01-02", "UTC"), + "timestamp": entry.time.number(), + "open": entry.open.number(), + "high": entry.high.number(), + "low": entry.low.number(), + "close": entry.close.number(), + "volume_from": entry.volumefrom.number(), + "volume_to": entry.volumeto.number() + }) + } else { [] } + }, + "metadata": { + "source": "CryptoCompare API", + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z", "UTC"), + "api_response": this.Response + } + } + } else { + { + "success": false, + "error": "API request failed: " + this.Response.format_json(), + "metadata": { + "source": "CryptoCompare API", + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z", "UTC") + } + } + } + +meta: + mcp: + enabled: true + description: "Fetches Bitcoin price history from CryptoCompare API for the specified number of days (default 30)" + properties: + - name: days + type: number + description: "Number of days of historical data to fetch (default: 30, max: 2000)" + required: false + - name: currency + type: string + description: "Target currency for price data (default: USD)" + required: false +---- +<1> **label:** Names this processor config for identification. +<2> **prepare_parameters:** Sets up parameters (`days`, `currency`) with defaults and builds the API endpoint URL. +<3> **fetch_data:** Makes an HTTP GET request to the CryptoCompare API using the prepared endpoint. +<4> **format_response:** Formats the API response into a structured JSON object, handling both success and error cases. +==== + +. Click *Lint* to check the configuration. You should see no errors. + +. Click *Create MCP Server* to deploy the server. ++ +It may take a few seconds to start. the status changes from *Starting* to *Running* when it's ready. + +. Open *MCP Inspector* tab to test the tool. Enter the request parameters such as `days=1` and `currency=USD`. Click *Run Tool*. You should see a response with Bitcoin price data. + +== Connect a client + +You can connect any MCP-compatible AI client to your remote MCP server. This example uses Claude Code. + +The Redpanda CLI (`rpk`) acts as a local proxy, forwarding requests from your AI client to your own remote MCP server running in the Redpanda Cloud cluster. + +. Log in to your Redpanda Cloud account: ++ +[,bash] +---- +rpk cloud login +---- ++ +This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. + +. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. It looks like this: ++ +[,bash] +---- +# Add to your Claude Desktop configuration +rpk cloud mcp proxy \ +--cluster-id \ +--mcp-server-id \ +--install --client claude-code +---- + +. Restart Claude Code and invoke your tool. ++ +[,bash] +---- +claude +---- + +. Ask Claude Code to run your tool. For example: ++ +* "Fetch Bitcoin price history for the last 10 days in EUR." +* "Get BTC price data for 60 days in USD." + +== Troubleshoot + +=== MCP server not starting + +If your server is not starting or shows an error state: + +- Check the *Logs* tab for specific error messages. +- Verify your YAML syntax by clicking *Lint*. +- Ensure all required secrets are created in the Secrets Store. + +=== Connection issues + +If Claude Code can't connect to your server: + +- Verify you're logged in with `rpk cloud login`. +- Check that your rpk version is 25.2.3 or later: `rpk version`. +- Ensure your server status shows *Running* in Redpanda Cloud. +- Try restarting Claude Code after running the proxy command. + +=== API request failures + +If the Bitcoin price tool returns errors: + +- The CryptoCompare API may be rate limiting requests. Wait a minute and try again. +- Check your internet connection. +- Use the MCP Inspector in Redpanda Cloud to test with different parameters. + +== Next steps + +* xref:ai-agents:mcp/remote/developer-guide.adoc[] diff --git a/modules/ai-agents/partials/beta.adoc b/modules/ai-agents/partials/beta.adoc new file mode 100644 index 000000000..cf01d3ce5 --- /dev/null +++ b/modules/ai-agents/partials/beta.adoc @@ -0,0 +1 @@ +NOTE: Remote MCP servers are in **private beta** and are free to use until they are released as Generally Available (GA). \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc index d94f56b26..286dc30dc 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc @@ -1,3 +1,4 @@ = rpk cloud mcp install +:description: Install the local MCP server for Redpanda Cloud configuration. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-install.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc new file mode 100644 index 000000000..8b30da916 --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc @@ -0,0 +1,4 @@ += rpk cloud mcp proxy +:description: Proxy MCP requests to remote MCP servers in Redpanda Cloud + +include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-proxy.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc index b440addd6..aa1f11f42 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc @@ -1,3 +1,4 @@ = rpk cloud mcp stdio +:description: Communicate with the local MCP server for Redpanda Cloud using the stdio protocol. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-stdio.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc new file mode 100644 index 000000000..0cbf45193 --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc @@ -0,0 +1,4 @@ += rpk cloud mcp +:description: Manage connections to MCP servers in Redpanda Cloud. + +include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp.adoc[tag=single-source] \ No newline at end of file From 23dcb5f9ee682cb061ad302a20d05899a7108b59 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:32:01 +0100 Subject: [PATCH 02/30] Update modules/ai-agents/pages/mcp/remote/developer-guide.adoc Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 8f7e09773..663ba1cf5 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -122,8 +122,6 @@ meta: properties: [] ---- - } - See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] ==== External API calls (processor patterns) From 4b20b55e930bd74190736e4b37769ab24b795a6c Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 16 Sep 2025 16:33:10 +0100 Subject: [PATCH 03/30] Add period --- modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc index 8b30da916..b38ce0fe2 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc @@ -1,4 +1,4 @@ = rpk cloud mcp proxy -:description: Proxy MCP requests to remote MCP servers in Redpanda Cloud +:description: Proxy MCP requests to remote MCP servers in Redpanda Cloud. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-proxy.adoc[tag=single-source] \ No newline at end of file From ecba07c0364a6845134878f5d46b0eb5ff0aec7e Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:12:35 +0100 Subject: [PATCH 04/30] Apply suggestions from code review --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 663ba1cf5..7ae17470c 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -24,7 +24,7 @@ Redpanda Connect:: A framework for building event-driven data pipelines. Tools i Tool:: A single request/response operation exposed to MCP. Each tool is implemented as a Redpanda Connect pipeline and described to MCP with `meta.mcp`. -Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use *UPPER_CASE snake_case* for secret names. +Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as DATAPLANE_TOKEN). == Development workflow @@ -73,7 +73,7 @@ All credentials and sensitive values must be stored in the Redpanda Cloud xref:d - Reference secrets as `${secrets.NAME}` in your pipeline YAML. - Never commit secrets to Git or reference them directly inline in configuration files. -- Use `UPPER_CASE` snake_case for secret names (such as `DATAPLANE_TOKEN`). +- Use uppercase snakecase for secret names (such as DATAPLANE_TOKEN). - Rotate secrets in the Secrets Store as needed. - Only request the scopes/roles required by your tool (principle of least privilege). From 211d5c70dae15147f908222876b7e1353cb260a4 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:39:48 +0100 Subject: [PATCH 05/30] Update modules/ai-agents/pages/mcp/remote/developer-guide.adoc --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 7ae17470c..6fe1abbc9 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -816,7 +816,7 @@ When your pipeline is ready, deploy it as a remote MCP server in Redpanda Cloud . Log into your Redpanda Cloud Console at https://cloud.redpanda.com. . Select your cluster from the cluster list. . In the left sidebar, navigate to *Remote MCP*. -. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button +. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. === Create and deploy your MCP server From 6aac95b9c7a331b3398e145705ce22087edd8ce0 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 16 Sep 2025 17:44:22 +0100 Subject: [PATCH 06/30] Update nav --- modules/ROOT/nav.adoc | 3 ++- modules/ai-agents/pages/mcp/remote/index.adoc | 4 ++++ modules/ai-agents/pages/mcp/remote/overview.adoc | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 modules/ai-agents/pages/mcp/remote/index.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 36a1432d6..9c2450025 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -75,7 +75,8 @@ *** xref:ai-agents:mcp/local/overview.adoc[Overview] *** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/local/configuration.adoc[Configure] -** xref:ai-agents:mcp/remote/overview.adoc[Remote MCP] +** xref:ai-agents:mcp/remote/index.adoc[Remote MCP] +** xref:ai-agents:mcp/remote/overview.adoc[Overview] *** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] *** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] diff --git a/modules/ai-agents/pages/mcp/remote/index.adoc b/modules/ai-agents/pages/mcp/remote/index.adoc new file mode 100644 index 000000000..1b7c481a3 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/index.adoc @@ -0,0 +1,4 @@ += Remote MCP Servers for Redpanda Cloud +:page-beta: true +:description: Find links to information about the Remote MCP servers for Redpanda Cloud and features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. +:page-layout: index diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index 829a536b1..36da1062c 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -1,4 +1,4 @@ -= Remote MCP Servers for Redpanda Cloud += About Remote MCP Servers for Redpanda Cloud :page-beta: true :description: Learn about remote MCP servers, which are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. From 20e60cf7a16dddd941f183d91ef6816d0bdbdd5e Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 16 Sep 2025 17:44:57 +0100 Subject: [PATCH 07/30] Update nav --- modules/ROOT/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 9c2450025..2d23b7fb0 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -71,7 +71,7 @@ * xref:ai-agents:index.adoc[AI Agents] ** xref:ai-agents:mcp/overview.adoc[MCP Overview] -** xref:ai-agents:mcp/local/index.adoc[Local MCP server for Redpanda Cloud] +** xref:ai-agents:mcp/local/index.adoc[Local Redpanda Cloud MCP] *** xref:ai-agents:mcp/local/overview.adoc[Overview] *** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/local/configuration.adoc[Configure] From 5a1d8eee83681f677d24da5ef2e5a315a49205dc Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:09:01 +0100 Subject: [PATCH 08/30] Apply suggestions from code review --- modules/ROOT/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 2d23b7fb0..e3c8b0268 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -76,7 +76,7 @@ *** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/local/configuration.adoc[Configure] ** xref:ai-agents:mcp/remote/index.adoc[Remote MCP] -** xref:ai-agents:mcp/remote/overview.adoc[Overview] +*** xref:ai-agents:mcp/remote/overview.adoc[Overview] *** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] *** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] From 39981312a64468c7dfc8d3ad365c52d25b1dabf0 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Wed, 17 Sep 2025 10:14:23 +0100 Subject: [PATCH 09/30] Simplify --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 6fe1abbc9..e70c103c3 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -334,11 +334,6 @@ processors: meta user_id = this.user_id.string() meta limit = this.limit.number().catch(10) meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) - - # Range validation - meta validated_limit = if @limit < 1 || @limit > 1000 { - throw("limit must be between 1 and 1000") - } else { @limit } ---- ==== Dynamic configuration @@ -437,9 +432,7 @@ processors: Accept: "application/json" ---- + -<1> The secret is injected at runtime. Never store the actual API key in your YAML - -IMPORTANT: The `${secrets.NAME}` syntax is resolved by Redpanda Connect at runtime. The actual secret value never appears in your configuration files or logs. +<1> The secret is injected at runtime. Never store the actual API key in your YAML. The actual secret value never appears in your configuration files or logs. ==== Monitoring, debugging, and observability From 19be341cbfe07df1b0d11416dc39704139e4c32e Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:17:07 +0100 Subject: [PATCH 10/30] Update configuration.adoc --- modules/ai-agents/pages/mcp/local/configuration.adoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/ai-agents/pages/mcp/local/configuration.adoc b/modules/ai-agents/pages/mcp/local/configuration.adoc index 22861fcb2..7b4453f3c 100644 --- a/modules/ai-agents/pages/mcp/local/configuration.adoc +++ b/modules/ai-agents/pages/mcp/local/configuration.adoc @@ -175,10 +175,6 @@ Expected response shapes (examples): + .. Stop the server with `Ctrl+C`. -== Common issues - -The server runs locally, but it must reach Redpanda Cloud endpoints to perform operations. - === Client can't find the MCP server * Re-run the install for your MCP client. From f9aab5239db9c007968c85ed7afea2042910f264 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:18:41 +0100 Subject: [PATCH 11/30] Update overview.adoc --- .../ai-agents/pages/mcp/remote/overview.adoc | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index 36da1062c..a35044dbd 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -18,27 +18,6 @@ Remote MCP servers are managed, hosted Model Context Protocol servers that run i * *Host*: Redpanda Cloud runs your server within the target cluster using the resources you select. * *Access*: Clients such as Claude Code connect and authenticate using either the MCP proxy or direct connections. -== Core concepts - -The remote MCP server is a managed service that runs inside your Redpanda Cloud cluster. You create and deploy the server using the Redpanda Cloud Console or the API. - -=== Redpanda Connect - -Remote MCP servers are implemented as Redpanda Connect pipelines. Your tools: - -* Use processors for request/response logic. -* Can access brokers using the `$\{REDPANDA_BROKERS}` environment variable (the pipeline is hosted within the cluster). - -=== Secrets - -* Store secrets in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] and reference them in configs as `${secrets.MY_SECRET_NAME}`. -* *Never* include secrets in plain text within pipelines or MCP server definitions. -* Secret names should be *UPPER_CASE snake_case*. - -=== Linting and validation - -Always use the *Lint* action before deploying to ensure the configuration compiles and required parameters are present. - == Use cases [cols="1,3a"] From a32f397ed16854d26d8c767af9481fbee5caa2d1 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Wed, 17 Sep 2025 10:25:24 +0100 Subject: [PATCH 12/30] Update headings to be task oriented --- .../pages/mcp/remote/developer-guide.adoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index e70c103c3..2d23f0eee 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -82,13 +82,13 @@ See an example of using secrets in the <> section. [[pipeline-patterns]] == Pipeline patterns -Remote MCP tools are implemented as Redpanda Connect pipelines. This section shows you how to build different types of MCP tools using various pipeline patterns, organized from basic to advanced. +Remote MCP tools are implemented as Redpanda Connect pipelines. This section shows you how to build different types of MCP tools using various pipeline patterns, organized by common use cases and integration scenarios. TIP: For detailed configuration options for any component, see the xref:develop:connect/components/about.adoc[Redpanda Connect components reference]. -=== Basic patterns +=== Data generation and simple requests -These patterns cover the most common MCP tool use cases. Start with these if you're new to building remote MCP tools. +Start here if you're building your first MCP tools. These patterns cover data generation, HTTP API calls, and basic database queries. ==== Data generators (input patterns) @@ -226,9 +226,9 @@ meta: See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) -=== Intermediate patterns +=== Redpanda integration and data publishing -These patterns handle more complex scenarios like data publishing, caching, and topic inspection. Use these when you need to integrate with Redpanda infrastructure or handle stateful operations. +Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. ==== Data publishers (output patterns) @@ -312,9 +312,9 @@ meta: See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] -=== Advanced patterns +=== Production workflows and observability -These patterns handle complex scenarios like multi-step workflows, production observability, and sophisticated error handling. Use these when building enterprise-grade tools. +Build enterprise-grade tools with error handling, validation, multi-step workflows, and monitoring. ==== Parameter validation and type coercion From 1de6026a756c530a69fecf70de1a0bf5689a69da Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Wed, 17 Sep 2025 11:30:43 +0100 Subject: [PATCH 13/30] Require 25.2.5 --- .../pages/mcp/remote/developer-guide.adoc | 27 ++++++++++++++----- .../pages/mcp/remote/quickstart.adoc | 17 +++++++++--- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 2d23f0eee..0fa5dcbb6 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -10,6 +10,7 @@ This guide provides a reference for building remote MCP servers that are managed * Access to a Redpanda Cloud cluster * Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries +* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] * (Optional) Claude / Claude Code for testing TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. @@ -863,18 +864,30 @@ rpk cloud login + This opens a browser window for sign-in and stores your authentication token locally. -. Install the MCP proxy for your client (Claude/Claude Code): +. Install the MCP proxy for your client (Claude/Claude Code). + -[source,bash] +For BYOC and Dedicated clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--cluster-id \ <1> +--mcp-server-id \ <2> +--install --client claude-code +---- ++ +For Serverless clusters, use: ++ +[,bash] ---- rpk cloud mcp proxy \ - --cluster-id \ <1> - --mcp-server-id \ <2> - --install --client claude-code +--serverless-cluster-id \ <1> +--mcp-server-id \ <2> +--install --client claude-code ---- + -<1> `--cluster-id`: The target Redpanda Cloud cluster ID where your remote MCP server is hosted. -<2> `--mcp-server-id`: The unique ID of your deployed remote MCP server +<1> The target Redpanda Cloud cluster ID where your remote MCP server is hosted. +<2> The unique ID of your deployed remote MCP server + TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. + diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 4adf883d4..80470faa6 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -10,7 +10,7 @@ include::ai-agents:partial$beta.adoc[] * A Redpanda Cloud cluster and subscription with *Remote MCP* enabled * Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials (if needed) -* At least version 25.2.3 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] +* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] * link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. @@ -142,16 +142,25 @@ rpk cloud login + This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. -. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. It looks like this: +. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. It looks like this for BYOC and Dedicated clusters: + [,bash] ---- -# Add to your Claude Desktop configuration rpk cloud mcp proxy \ --cluster-id \ --mcp-server-id \ --install --client claude-code ---- ++ +For Serverless clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--serverless-cluster-id \ +--mcp-server-id \ +--install --client claude-code +---- . Restart Claude Code and invoke your tool. + @@ -180,7 +189,7 @@ If your server is not starting or shows an error state: If Claude Code can't connect to your server: - Verify you're logged in with `rpk cloud login`. -- Check that your rpk version is 25.2.3 or later: `rpk version`. +- Check that your rpk version is 25.2.5 or later: `rpk version`. - Ensure your server status shows *Running* in Redpanda Cloud. - Try restarting Claude Code after running the proxy command. From cd974a8efc88c93c9317bb2f8baebb8128d84071 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 18 Sep 2025 17:56:38 +0100 Subject: [PATCH 14/30] Improve overview --- .../ai-agents/pages/mcp/remote/overview.adoc | 121 ++++++++++++++++-- 1 file changed, 112 insertions(+), 9 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index a35044dbd..f4e7284fa 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -4,19 +4,89 @@ include::ai-agents:partial$beta.adoc[] -Remote MCP servers are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. Unlike the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud], remote servers are deployed to your cluster and operated by Redpanda. Your server can include one or more tools that you write using Redpanda Connect pipelines. +Remote MCP servers are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. Unlike the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud], remote servers are deployed to your cluster and operated by Redpanda. + +Remote MCP servers enable you to expose your custom data pipelines as MCP tools (AI-consumable endpoints). MCP provides a standardized way for AI systems to interact with tools and services. Learn more about MCP in the link:https://docs.anthropic.com/en/docs/mcp[official documentation^]. + +== Understanding tools and pipelines + +In Redpanda Connect, pipelines are declarative YAML files composed of inputs, processors, and outputs. In MCP, these pipelines become *tools*, which are discrete units of functionality that AI systems can discover and execute through the MCP interface. + +AI agents can perform complex tasks, but they need actionable interfaces to interact with real systems. Remote MCP servers provide a simple way to expose your internal tools as endpoints that AI systems can understand and use. + +Each tool is defined as a small, independent unit with: + +* A unique `label` for identification +* A plain-English `description` that LLMs can understand +* Structured `properties` that define inputs and outputs +* Pipeline logic using Redpanda Connect processors to perform the actual work + +For example, a system that fetches customer data, analyzes streaming metrics, or generates reports can be exposed as a tool that AI assistants can invoke using natural language with zero prompt engineering. == Why use remote MCP? * *Always-on and centrally managed:* No need to run a local process. Your tools live with your data. * *Proximity to data:* Execute tools next to your cluster for lower latency and simpler networking. * *Secure by design:* Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in pipelines. +* *Fast iteration:* Define tools as YAML files, deploy your server, and your AI agents can use the new logic instantly. -== Architecture +== Key features + +[cols="1s,3"] +|=== +|Feature |Description + +|Composable tools +|Each tool runs independently. They can be composed by AI into chains or workflows. + +|Minimal surface area +|Expose only the tools you want, reducing the risk of confusing your AI agents. + +|Cluster integration +|Direct access to your cluster's topics, schemas, and metrics without external networking. + +|Managed hosting +|Redpanda operates your MCP server with automatic scaling and monitoring. +|=== -* *Build*: You describe tools as Redpanda Connect components with `meta.mcp` metadata. -* *Host*: Redpanda Cloud runs your server within the target cluster using the resources you select. -* *Access*: Clients such as Claude Code connect and authenticate using either the MCP proxy or direct connections. +== Example: Customer analytics tool + +Here's how you might expose a customer analytics pipeline as an MCP tool: + +[source,yaml] +---- +label: analyze_customer_orders +processors: + - label: fetch_customer_data + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "orders" + where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" + args_mapping: 'root = [this.customer_id]' + + - label: calculate_metrics + mutation: | + root = { + "customer_id": this.customer_id, + "total_orders": this.length(), + "total_spent": this.map_each(o -> o.total).sum(), + "avg_order_value": this.map_each(o -> o.total).mean(), + "last_order_date": this.map_each(o -> o.created_at).max() + } + +meta: + mcp: + enabled: true + description: "Analyze a customer's order history and spending patterns over the last 30 days" + properties: + - name: customer_id + type: string + description: "Customer ID to analyze" + required: true +---- + +This tool can be invoked by an AI assistant with prompts like "analyze the order history for customer ID 12345" without needing to know the underlying SQL or processing logic. == Use cases @@ -47,10 +117,43 @@ Remote MCP servers are managed, hosted Model Context Protocol servers that run i == How it works -. **Authentication:** The AI client (such as Claude Code) must authenticate to Redpanda Cloud and receive a token. You can do this by using either the MCP proxy or by implementing the MCP client flow yourself. -.. *MCP proxy*: For tools with built-in MCP clients, first authenticate with `rpk cloud login`, then configure them to use the `rpk cloud mcp proxy` command. This command handles proxying requests to the remote MCP server running in your Redpanda Cloud cluster using your stored authentication token. -.. **Direct option:** If you are building your own AI agent, you can implement the MCP client flow yourself and connect directly to the remote MCP server, handling authentication and requests without `rpk`. -. **Prompting:** After connecting, you can prompt your assistant to perform operations using your remote MCP server. +Remote MCP servers follow a three-step process: + +. Build your tools. ++ +Define tools as Redpanda Connect pipelines with `meta.mcp` metadata. Each tool specifies: ++ +* **Description**: Plain-English explanation of what the tool does. +* **Properties**: Structured input parameters with types and validation. +* **Pipeline logic**: Processors that handle the actual work (API calls, database queries, data transformations). + +. Deploy to your cluster. ++ +Redpanda Cloud hosts your MCP server inside your cluster with: ++ +* **Managed infrastructure**: Automatic scaling, monitoring, and updates. +* **Resource allocation**: Choose CPU/memory based on your tool requirements. +* **Secure execution**: Access to cluster resources and the Secrets Store. + +. Connect your AI client. ++ +AI assistants connect through authentication and discovery: ++ +* **Authentication**: Use `rpk cloud login` followed by `rpk cloud mcp proxy` for automatic setup. +* **Tool discovery**: Your AI client automatically discovers available tools and their capabilities. +* **Natural language execution**: Invoke tools through conversational prompts. + +== Authentication options + +- **MCP proxy (recommended)**: For tools with built-in MCP clients like Claude Code: ++ +[,bash] +---- +rpk cloud login +rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id +---- + +- **Direct connection**: If building your own AI agent, implement the MCP client flow yourself and connect directly to the remote MCP server. == Next steps From d8e206b940ed8854cb1e2685ef9af40b5fc332b1 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Fri, 19 Sep 2025 16:02:50 +0100 Subject: [PATCH 15/30] Improvements --- .../pages/mcp/remote/developer-guide.adoc | 111 ++++++++++++--- .../ai-agents/pages/mcp/remote/overview.adoc | 128 +++++------------- 2 files changed, 132 insertions(+), 107 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 0fa5dcbb6..dabb8b1d1 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -11,7 +11,7 @@ This guide provides a reference for building remote MCP servers that are managed * Access to a Redpanda Cloud cluster * Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries * At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] -* (Optional) Claude / Claude Code for testing +* (Optional) An AI assistant like Claude or Claude Code for testing TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. @@ -29,6 +29,7 @@ Secrets:: Credentials and tokens must be stored in the Secrets Store and referen == Development workflow +. <>: Select a Redpanda Cloud cluster to host your MCP server. . <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. . <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. . <>: Build an MCP tool using Redpanda Connect pipelines. See <> for examples of what is allowed in a pipeline. @@ -36,6 +37,19 @@ Secrets:: Credentials and tokens must be stored in the Secrets Store and referen . <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. . <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. +[[access]] +== Access MCP Servers in Redpanda Cloud + +. Log into your link:https://cloud.redpanda.com[Redpanda Cloud Console^]. +. Select your cluster from the cluster list. +. In the left sidebar, navigate to *Remote MCP*. +. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. +. Fill in the required fields: ++ +* *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). +* *Description*: Brief description of what your server does. +* *Configuration*: This is where to put your YAML configuration to define the an MCP tool. + [[contract]] == Tool contract and MCP metadata @@ -67,6 +81,8 @@ Property guidance: * Mark only mandatory fields as required. * Document defaults in the `description` and enforce them in the pipeline. +After defining the contract, implement the pipeline logic using Redpanda Connect processors, inputs, outputs, and caches. + [[provision-secrets]] == Provision secrets @@ -87,6 +103,83 @@ Remote MCP tools are implemented as Redpanda Connect pipelines. This section sho TIP: For detailed configuration options for any component, see the xref:develop:connect/components/about.adoc[Redpanda Connect components reference]. +=== YAML configuration rules + +Each YAML configuration (tool) should contain exactly one component type. The component type is inferred from the type you choose in the dropdown when creating or editing the MCP server. Valid component types are: + +* `input` (for data sources) +* `output` (for data sinks) +* `processor` (for data transformations) +* `cache` (for caching intermediate results) + +.Correct example for inputs +[source,yaml] +---- +label: my_input +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "events" ] + consumer_group: "mcp-reader" + +meta: + mcp: + enabled: true + description: "Consume events from Redpanda" +---- + +.Correct example for processors +[source,yaml] +---- +processors: + - label: safe_operation + try: + - http: + url: "https://api.example.com/data" + timeout: "10s" + - mutation: | + root = this.merge({"processed": true}) + + - label: handle_errors + catch: + - mutation: | + root = { + "error": "Operation failed", + "details": error() + } +---- + +.Incorrect (do not include the input wrapper) +[source,yaml] +---- +label: my_input +input: + redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "events" ] +---- + +.Incorrect (multiple component types in one file) +[source,yaml] +---- +input: + redpanda: { ... } +processors: + - mutation: { ... } +output: + redpanda: { ... } +---- + +.Incorrect (try/catch as single processor) +[source,yaml] +---- +processors: + - label: operation + try: + - http: { ... } + catch: + - mutation: { ... } +---- + === Data generation and simple requests Start here if you're building your first MCP tools. These patterns cover data generation, HTTP API calls, and basic database queries. @@ -805,22 +898,8 @@ Notice how the workflow: When your pipeline is ready, deploy it as a remote MCP server in Redpanda Cloud and test it with the built-in MCP Inspector. -=== Access MCP Servers in Redpanda Cloud - -. Log into your Redpanda Cloud Console at https://cloud.redpanda.com. -. Select your cluster from the cluster list. -. In the left sidebar, navigate to *Remote MCP*. -. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. - -=== Create and deploy your MCP server +=== Deploy your MCP server -. Click *Create MCP Server* to open the creation form. -. Fill in the required fields: -+ -* *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). -* *Description*: Brief description of what your server does. -* *Configuration*: Paste your YAML pipeline configuration. -. Click *Lint* to check your YAML syntax and configuration. . Click *Create MCP Server* to deploy the server. . Monitor the *Status* column. It will show one of: + diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index f4e7284fa..98af3e5e5 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -1,53 +1,54 @@ = About Remote MCP Servers for Redpanda Cloud :page-beta: true -:description: Learn about remote MCP servers, which are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. +:description: Business value, use cases, and architecture for remote MCP servers in Redpanda Cloud. include::ai-agents:partial$beta.adoc[] -Remote MCP servers are managed, hosted Model Context Protocol servers that run inside your Redpanda Cloud cluster. Unlike the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud], remote servers are deployed to your cluster and operated by Redpanda. +Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that you can write and run inside your Redpanda Cloud cluster. They let you expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: -Remote MCP servers enable you to expose your custom data pipelines as MCP tools (AI-consumable endpoints). MCP provides a standardized way for AI systems to interact with tools and services. Learn more about MCP in the link:https://docs.anthropic.com/en/docs/mcp[official documentation^]. +* Make your data and workflows accessible to AI assistants and LLMs, safely and on your terms. +* Accelerate automation and insight by letting AI trigger real business actions. +* Control exposure so that only the tools you tag are visible to AI. -== Understanding tools and pipelines +== Why use a remote MCP server? -In Redpanda Connect, pipelines are declarative YAML files composed of inputs, processors, and outputs. In MCP, these pipelines become *tools*, which are discrete units of functionality that AI systems can discover and execute through the MCP interface. +* *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. +* *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. +* *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in pipelines. +* *Fast iteration*: Define tools as YAML configuration, deploy your MCP server, and your AI agents can use the new logic instantly. -AI agents can perform complex tasks, but they need actionable interfaces to interact with real systems. Remote MCP servers provide a simple way to expose your internal tools as endpoints that AI systems can understand and use. - -Each tool is defined as a small, independent unit with: - -* A unique `label` for identification -* A plain-English `description` that LLMs can understand -* Structured `properties` that define inputs and outputs -* Pipeline logic using Redpanda Connect processors to perform the actual work +== Use cases -For example, a system that fetches customer data, analyzes streaming metrics, or generates reports can be exposed as a tool that AI assistants can invoke using natural language with zero prompt engineering. +[cols="1s,3a"] +|=== +|Category |Example prompts -== Why use remote MCP? +|Operational monitoring +|* Check partition lag for customer-events topic +* Show me the top 10 producers by message volume today +* Get schema registry health status -* *Always-on and centrally managed:* No need to run a local process. Your tools live with your data. -* *Proximity to data:* Execute tools next to your cluster for lower latency and simpler networking. -* *Secure by design:* Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in pipelines. -* *Fast iteration:* Define tools as YAML files, deploy your server, and your AI agents can use the new logic instantly. +|Data enrichment and analysis +|* Fetch user profile data and recent orders for customer ID 12345 +* Get real-time stock prices for symbols in my portfolio topic +* Analyze sentiment of latest product reviews -== Key features +|Team productivity +|* Deploy my microservice to the staging environment +* Generate load test data for the payments service +* Create a summary dashboard of this week's incident reports -[cols="1s,3"] +|Business intelligence +|* What are the trending products in the last 24 hours? +* Show revenue impact of the latest feature deployment +* Get customer satisfaction scores from support tickets |=== -|Feature |Description - -|Composable tools -|Each tool runs independently. They can be composed by AI into chains or workflows. -|Minimal surface area -|Expose only the tools you want, reducing the risk of confusing your AI agents. - -|Cluster integration -|Direct access to your cluster's topics, schemas, and metrics without external networking. +== How it works -|Managed hosting -|Redpanda operates your MCP server with automatic scaling and monitoring. -|=== +. Build your tools as Redpanda Connect pipelines with `meta.mcp` metadata (description, properties, and logic). +. Deploy to your cluster. Redpanda Cloud hosts your MCP server with secure access. +. Connect your AI client. AI assistants authenticate, discover available tools, and invoke them using natural language prompts. == Example: Customer analytics tool @@ -88,64 +89,9 @@ meta: This tool can be invoked by an AI assistant with prompts like "analyze the order history for customer ID 12345" without needing to know the underlying SQL or processing logic. -== Use cases - -[cols="1,3a"] -|=== -|Category |Example prompts - -|Operational monitoring -|* Check partition lag for customer-events topic -* Show me the top 10 producers by message volume today -* Get schema registry health status - -|Data enrichment and analysis -|* Fetch user profile data and recent orders for customer ID 12345 -* Get real-time stock prices for symbols in my portfolio topic -* Analyze sentiment of latest product reviews - -|Team productivity -|* Deploy my microservice to the staging environment -* Generate load test data for the payments service -* Create a summary dashboard of this week's incident reports - -|Business intelligence -|* What are the trending products in the last 24 hours? -* Show revenue impact of the latest feature deployment -* Get customer satisfaction scores from support tickets -|=== - -== How it works - -Remote MCP servers follow a three-step process: - -. Build your tools. -+ -Define tools as Redpanda Connect pipelines with `meta.mcp` metadata. Each tool specifies: -+ -* **Description**: Plain-English explanation of what the tool does. -* **Properties**: Structured input parameters with types and validation. -* **Pipeline logic**: Processors that handle the actual work (API calls, database queries, data transformations). - -. Deploy to your cluster. -+ -Redpanda Cloud hosts your MCP server inside your cluster with: -+ -* **Managed infrastructure**: Automatic scaling, monitoring, and updates. -* **Resource allocation**: Choose CPU/memory based on your tool requirements. -* **Secure execution**: Access to cluster resources and the Secrets Store. - -. Connect your AI client. -+ -AI assistants connect through authentication and discovery: -+ -* **Authentication**: Use `rpk cloud login` followed by `rpk cloud mcp proxy` for automatic setup. -* **Tool discovery**: Your AI client automatically discovers available tools and their capabilities. -* **Natural language execution**: Invoke tools through conversational prompts. - == Authentication options -- **MCP proxy (recommended)**: For tools with built-in MCP clients like Claude Code: +- **MCP proxy (recommended):** For tools with built-in MCP clients like Claude Code: + [,bash] ---- @@ -153,9 +99,9 @@ rpk cloud login rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id ---- -- **Direct connection**: If building your own AI agent, implement the MCP client flow yourself and connect directly to the remote MCP server. +- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the remote MCP server. -== Next steps +== Next Steps * xref:ai-agents:mcp/remote/quickstart.adoc[] * xref:ai-agents:mcp/remote/developer-guide.adoc[] From e8816ea582e90c909e8708bde0fb79f64ac91e13 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 22 Sep 2025 11:59:05 +0100 Subject: [PATCH 16/30] Split examples --- modules/ROOT/nav.adoc | 1 + .../examples/customer_enrichment.yaml | 39 + .../gcp_bigquery_select_processor.yaml | 45 + .../ai-agents/examples/generate_input.yaml | 22 + .../ai-agents/examples/http_processor.yaml | 37 + modules/ai-agents/examples/memory_cache.yaml | 14 + .../ai-agents/examples/observable_tool.yaml | 60 ++ .../ai-agents/examples/order_workflow.yaml | 107 +++ .../ai-agents/examples/redpanda_cache.yaml | 15 + .../ai-agents/examples/redpanda_output.yaml | 25 + .../ai-agents/examples/weather_service.yaml | 60 ++ .../pages/mcp/remote/developer-guide.adoc | 791 +----------------- .../ai-agents/pages/mcp/remote/overview.adoc | 12 +- .../pages/mcp/remote/pipeline-patterns.adoc | 306 +++++++ 14 files changed, 786 insertions(+), 748 deletions(-) create mode 100644 modules/ai-agents/examples/customer_enrichment.yaml create mode 100644 modules/ai-agents/examples/gcp_bigquery_select_processor.yaml create mode 100644 modules/ai-agents/examples/generate_input.yaml create mode 100644 modules/ai-agents/examples/http_processor.yaml create mode 100644 modules/ai-agents/examples/memory_cache.yaml create mode 100644 modules/ai-agents/examples/observable_tool.yaml create mode 100644 modules/ai-agents/examples/order_workflow.yaml create mode 100644 modules/ai-agents/examples/redpanda_cache.yaml create mode 100644 modules/ai-agents/examples/redpanda_output.yaml create mode 100644 modules/ai-agents/examples/weather_service.yaml create mode 100644 modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index bf4dd56a1..89858aee9 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -80,6 +80,7 @@ *** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] *** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] *** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] +*** xref:ai-agents:mcp/remote/pipeline-patterns.adoc[MCP Server Patterns] * xref:develop:connect/about.adoc[Redpanda Connect] ** xref:develop:connect/connect-quickstart.adoc[Quickstart] diff --git a/modules/ai-agents/examples/customer_enrichment.yaml b/modules/ai-agents/examples/customer_enrichment.yaml new file mode 100644 index 000000000..209624818 --- /dev/null +++ b/modules/ai-agents/examples/customer_enrichment.yaml @@ -0,0 +1,39 @@ +label: customer_enrichment +processors: + - label: fetch_customer_base + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "customers" + where: "customer_id = ?" + args_mapping: 'root = [this.customer_id]' + + - label: enrich_with_orders + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "orders" + where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" + args_mapping: 'root = [this.customer_id]' + + - label: combine_data + mutation: | + root = { + "customer": this.customers.index(0), + "recent_orders": this.orders, + "metrics": { + "total_orders": this.orders.length(), + "total_spent": this.orders.map_each(o -> o.total).sum(), + "avg_order_value": this.orders.map_each(o -> o.total).mean() + } + } + +meta: + mcp: + enabled: true + description: "Get comprehensive customer profile with recent order history and metrics" + properties: + - name: customer_id + type: string + description: "Customer ID to analyze" + required: true \ No newline at end of file diff --git a/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml b/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml new file mode 100644 index 000000000..c6620580d --- /dev/null +++ b/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml @@ -0,0 +1,45 @@ +label: gcp_bigquery_select_processor +processors: + - label: prepare_parameters + mutation: | + meta customer_id = this.customer_id.string().catch("12345") + meta limit = this.limit.number().catch(10) + - label: query_bigquery + gcp_bigquery_select: + project: my-gcp-project + credentials_json: | + ${secrets.BIGQUERY_CREDENTIALS} + table: my_dataset.customer_orders + columns: + - "order_id" + - "customer_id" + - "order_date" + - "total_amount" + - "status" + where: customer_id = ? AND order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) + suffix: "ORDER BY order_date DESC LIMIT ?" + args_mapping: root = [ @customer_id, @limit ] + - label: format_response + mutation: | + root = { + "orders": this, + "metadata": { + "source": "BigQuery", + "customer_id": @customer_id, + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Query customer orders from BigQuery" + properties: + - name: customer_id + type: string + description: "Customer ID to filter orders" + required: true + - name: limit + type: number + description: "Maximum number of orders to return" + required: false diff --git a/modules/ai-agents/examples/generate_input.yaml b/modules/ai-agents/examples/generate_input.yaml new file mode 100644 index 000000000..0c8106144 --- /dev/null +++ b/modules/ai-agents/examples/generate_input.yaml @@ -0,0 +1,22 @@ +label: generate_input +generate: + mapping: | + let event_type = ["login", "logout", "purchase", "view_page", "click_button"].index(random_int(max:4)) + root = { + "id": uuid_v4(), + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "user_id": random_int(min:1, max:10000), + "event_type": $event_type, + "data": { + "session_id": ksuid(), + "ip_address": "192.168.%v.%v".format(random_int(max:255), random_int(min:1, max:254)), + "user_agent": ["Chrome", "Firefox", "Safari", "Edge"].index(random_int(max:3)), + "amount": if $event_type == "purchase" { random_int(min:10, max:500) } else { null } + } + } + +meta: + mcp: + enabled: true + description: "Generate an example user event message with realistic data" + properties: [] diff --git a/modules/ai-agents/examples/http_processor.yaml b/modules/ai-agents/examples/http_processor.yaml new file mode 100644 index 000000000..6e523a1b3 --- /dev/null +++ b/modules/ai-agents/examples/http_processor.yaml @@ -0,0 +1,37 @@ +label: fetch-weather +processors: + - label: prepare_parameters + mutation: | + meta city_name = this.city_name + - label: fetch_weather + http: + url: 'https://wttr.in/${! @city_name }?format=j1' + verb: GET + headers: + Accept: "application/json" + User-Agent: "redpanda-mcp-server/1.0" + - label: format_response + mutation: | + root = { + "city": @city_name, + "temperature": this.current_condition.0.temp_C.number(), + "feels_like": this.current_condition.0.FeelsLikeC.number(), + "humidity": this.current_condition.0.humidity.number(), + "pressure": this.current_condition.0.pressure.number(), + "description": this.current_condition.0.weatherDesc.0.value, + "wind_speed": this.current_condition.0.windspeedKmph.number(), + "metadata": { + "source": "wttr.in", + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Fetch current weather information for a specified city" + properties: + - name: city_name + type: string + description: "Name of the city to get weather information for" + required: true diff --git a/modules/ai-agents/examples/memory_cache.yaml b/modules/ai-agents/examples/memory_cache.yaml new file mode 100644 index 000000000..bf67ab8ce --- /dev/null +++ b/modules/ai-agents/examples/memory_cache.yaml @@ -0,0 +1,14 @@ +label: memory_cache +memory: + default_ttl: "5m" + init_values: + "user:1001": '{"name": "Alice", "role": "admin"}' + "user:1002": '{"name": "Bob", "role": "user"}' + "config:theme": "dark" + "config:language": "en" + shards: 4 + +meta: + mcp: + enabled: true + description: "In-memory cache for storing user data, configuration, and temporary values" diff --git a/modules/ai-agents/examples/observable_tool.yaml b/modules/ai-agents/examples/observable_tool.yaml new file mode 100644 index 000000000..e2596a796 --- /dev/null +++ b/modules/ai-agents/examples/observable_tool.yaml @@ -0,0 +1,60 @@ +label: observable_tool +processors: + - label: init_tracing + mutation: | + # Generate correlation ID for request tracing + meta req_id = uuid_v7() + meta start_time = now() + + # Log request start with structured data + root.trace = { + "request_id": @req_id, + "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "version": "1.0.0" + } + + - label: log_request_start + log: + message: "MCP tool request started" + fields: + request_id: "${! @req_id }" + tool_name: "observable_tool" + input_params: "${! this.without(\"trace\") }" + user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }" + level: "INFO" + + - label: finalize_response + mutation: | + # Calculate total execution time + meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000 + + # Add trace information to response + root.metadata = { + "request_id": @req_id, + "execution_time_ms": @duration, + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "success": !this.exists("error") + } + + - label: log_completion + log: + message: "MCP tool request completed" + fields: + request_id: "${! @req_id }" + duration_ms: "${! this.metadata.execution_time_ms }" + success: "${! this.metadata.success }" + result_size: "${! content().length() }" + level: "INFO" + +meta: + tags: [ example ] + mcp: + enabled: true + description: "Example tool with comprehensive observability and error handling" + properties: + - name: user_id + type: string + description: "User ID to fetch data for" + required: true diff --git a/modules/ai-agents/examples/order_workflow.yaml b/modules/ai-agents/examples/order_workflow.yaml new file mode 100644 index 000000000..ad0dc29ee --- /dev/null +++ b/modules/ai-agents/examples/order_workflow.yaml @@ -0,0 +1,107 @@ +label: order_workflow +processors: + - label: validate_order + mutation: | + # Validation logic + root = if this.total <= 0 { + throw("Invalid order total") + } else { this } + + - label: mock_inventory_check + mutation: | + # Mock inventory data for testing + let inventory = { + "widget-001": {"quantity": 100, "name": "Standard Widget"}, + "widget-premium": {"quantity": 25, "name": "Premium Widget"}, + "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"} + } + + let product = $inventory.get(this.product_id) + root = if $product == null { + throw("Product not found: " + this.product_id) + } else if $product.quantity < this.quantity { + throw("Insufficient inventory. Available: " + $product.quantity.string()) + } else { + this.merge({ + "inventory_check": "passed", + "available_quantity": $product.quantity, + "product_name": $product.name + }) + } + + - label: route_by_priority + switch: + - check: 'this.total > 1000' + processors: + - label: mock_high_value_processing + mutation: | + # Mock premium processing + root = this.merge({ + "processing_tier": "premium", + "processing_time_estimate": "2-4 hours", + "assigned_rep": "premium-team@company.com", + "priority_score": 95 + }) + + - check: 'this.customer_tier == "vip"' + processors: + - label: mock_vip_processing + mutation: | + # Mock VIP processing + root = this.merge({ + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "assigned_rep": "vip-team@company.com", + "priority_score": 90, + "perks": ["expedited_shipping", "white_glove_service"] + }) + + - processors: + - label: mock_standard_processing + mutation: | + # Mock standard processing + root = this.merge({ + "processing_tier": "standard", + "processing_time_estimate": "24-48 hours", + "assigned_rep": "support@company.com", + "priority_score": 50 + }) + + - label: finalize_order + mutation: | + # Add final processing metadata + # Calculate estimated fulfillment by parsing processing time + let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number() + + root = this.merge({ + "order_status": "processed", + "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"), + "estimated_fulfillment": "TBD - calculated based on processing tier", + "processing_time_hours": $max_hours + }) + +meta: + mcp: + enabled: true + description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)" + properties: + - name: order_id + type: string + description: "Unique order identifier" + required: true + - name: product_id + type: string + description: "Product ID (try: widget-001, widget-premium, widget-limited)" + required: true + - name: quantity + type: number + description: "Quantity to order" + required: true + - name: total + type: number + description: "Order total in dollars" + required: true + - name: customer_tier + type: string + description: "Customer tier (optional: vip, standard)" + required: false \ No newline at end of file diff --git a/modules/ai-agents/examples/redpanda_cache.yaml b/modules/ai-agents/examples/redpanda_cache.yaml new file mode 100644 index 000000000..49be191d9 --- /dev/null +++ b/modules/ai-agents/examples/redpanda_cache.yaml @@ -0,0 +1,15 @@ +label: redpanda_cache +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] + topic: "mcp-cache-topic" + tls: + enabled: true + sasl: + - mechanism: "SCRAM-SHA-512" + username: "${secrets.MCP_REDPANDA_CREDENTIALS.username}" + password: "${secrets.MCP_REDPANDA_CREDENTIALS.password}" + +meta: + mcp: + enabled: true + description: "Redpanda-backed distributed cache using Kafka topics for persistence" diff --git a/modules/ai-agents/examples/redpanda_output.yaml b/modules/ai-agents/examples/redpanda_output.yaml new file mode 100644 index 000000000..324b135af --- /dev/null +++ b/modules/ai-agents/examples/redpanda_output.yaml @@ -0,0 +1,25 @@ +label: redpanda_output +redpanda: + seed_brokers: + - ${REDPANDA_BROKERS} + topic: ${! this.topic_name.string().catch("default-topic") } + timeout: 30s + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: ${secrets.REDPANDA_USERNAME} + password: ${secrets.REDPANDA_PASSWORD} +meta: + mcp: + enabled: true + description: Publishes a message to a specified Redpanda topic + properties: + - name: message + type: string + description: The message content to publish + required: true + - name: topic_name + type: string + description: The Redpanda topic to publish to + required: true diff --git a/modules/ai-agents/examples/weather_service.yaml b/modules/ai-agents/examples/weather_service.yaml new file mode 100644 index 000000000..1a6c56fe4 --- /dev/null +++ b/modules/ai-agents/examples/weather_service.yaml @@ -0,0 +1,60 @@ +label: weather-service +processors: + - label: validate_inputs + mutation: | + # Validate and sanitize city input + meta city = this.city.string(). + re_replace_all("[^a-zA-Z\\s\\-]", ""). + trim() + + # Check for empty input + root = if @city == "" { + throw("City name cannot be empty") + } else { "" } + + - label: fetch_weather_data + try: + - http: + url: "https://wttr.in/${! @city }?format=j1" + verb: GET + headers: + User-Agent: "redpanda-connect-mcp/1.0" + timeout: "10s" + - mutation: | + root = { + "city": @city, + "temperature_c": this.current_condition.0.temp_C.number(), + "temperature_f": this.current_condition.0.temp_F.number(), + "feels_like_c": this.current_condition.0.FeelsLikeC.number(), + "humidity": this.current_condition.0.humidity.number(), + "description": this.current_condition.0.weatherDesc.0.value, + "wind_speed_kmh": this.current_condition.0.windspeedKmph.number(), + "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") + } + - log: + message: "Weather data fetched for city: ${! @city }" + level: "INFO" + + - label: handle_weather_errors + catch: + - mutation: | + root = { + "error": "Failed to fetch weather data", + "city": @city, + "details": error(), + "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") + } + - log: + message: "Weather API error for city ${! @city }: ${! error() }" + level: "ERROR" + +meta: + tags: [ weather, example ] + mcp: + enabled: true + description: "Get current weather conditions for any city worldwide" + properties: + - name: city + type: string + description: "Name of the city (such as 'San Francisco', 'London')" + required: true diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index dabb8b1d1..889bb14df 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -4,7 +4,7 @@ include::ai-agents:partial$beta.adoc[] -This guide provides a reference for building remote MCP servers that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. +This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. == Prerequisites @@ -25,14 +25,14 @@ Redpanda Connect:: A framework for building event-driven data pipelines. Tools i Tool:: A single request/response operation exposed to MCP. Each tool is implemented as a Redpanda Connect pipeline and described to MCP with `meta.mcp`. -Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as DATAPLANE_TOKEN). +Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). == Development workflow . <>: Select a Redpanda Cloud cluster to host your MCP server. . <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. . <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. -. <>: Build an MCP tool using Redpanda Connect pipelines. See <> for examples of what is allowed in a pipeline. +. <>: Build an MCP tool using Redpanda Connect pipelines. . <>: Validate your pipeline, catch schema errors, then deploy and test with the MCP Inspector. . <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. . <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. @@ -51,7 +51,7 @@ Secrets:: Credentials and tokens must be stored in the Secrets Store and referen * *Configuration*: This is where to put your YAML configuration to define the an MCP tool. [[contract]] -== Tool contract and MCP metadata +== Design the tool contract and MCP metadata Each MCP tool must declare its interface using `meta.mcp` metadata. This metadata allows AI clients to discover and invoke the tool correctly. @@ -61,9 +61,9 @@ Define a clear, stable interface for each tool. Keep the description task-orient ---- meta: mcp: - enabled: true - description: "Fetches a compact summary from an external API using two optional parameters." - properties: + enabled: true <1> + description: "Fetches a compact summary from an external API using two optional parameters." <2> + properties: <3> - name: parameter1 type: string description: "Primary filter; defaults to provider standard when omitted." @@ -74,14 +74,18 @@ meta: required: false ---- +<1> Set `meta.mcp.enabled: true` to expose the tool using MCP. +<2> A concise description that explains what the tool does. This should be understandable by an AI model. +<3> List the input parameters (properties) for the tool. ++ Property guidance: - ++ * Use `string`, `number`, or `boolean` types. -* Validate ranges and enums using Bloblang inside the pipeline. +* Validate ranges and enums using xref:guides:bloblang/about.adoc[Bloblang] inside the pipeline. * Mark only mandatory fields as required. * Document defaults in the `description` and enforce them in the pipeline. -After defining the contract, implement the pipeline logic using Redpanda Connect processors, inputs, outputs, and caches. +After defining your tool contract, implement the pipeline logic to handle input validation, defaults, and the main processing steps. [[provision-secrets]] == Provision secrets @@ -90,32 +94,45 @@ All credentials and sensitive values must be stored in the Redpanda Cloud xref:d - Reference secrets as `${secrets.NAME}` in your pipeline YAML. - Never commit secrets to Git or reference them directly inline in configuration files. -- Use uppercase snakecase for secret names (such as DATAPLANE_TOKEN). +- Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). - Rotate secrets in the Secrets Store as needed. - Only request the scopes/roles required by your tool (principle of least privilege). -See an example of using secrets in the <> section. +See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[Pipeline patterns]. [[pipeline-patterns]] -== Pipeline patterns +== Implement the pipeline + +Use Redpanda Connect components to implement the logic of your MCP tool. Here are some best practices: + +* **Single responsibility**: Each tool should do one thing well. +* **Descriptive naming**: Use clear, specific labels like `fetch-user-profile` instead of generic names like `get-data`. The top-level label becomes the tool's name in the MCP server. +* **Input validation**: Always validate and sanitize user inputs using xref:develop:connect/guides/bloblang/about.adoc[Bloblang]. +* **Error handling**: Provide meaningful error messages. +* **Documentation**: Write clear descriptions that explain what the tool does and what it returns. + +For common patterns and examples, see xref:ai-agents:mcp/remote/pipeline-patterns.adoc[]. -Remote MCP tools are implemented as Redpanda Connect pipelines. This section shows you how to build different types of MCP tools using various pipeline patterns, organized by common use cases and integration scenarios. +Here's a complete example that demonstrates best practices: -TIP: For detailed configuration options for any component, see the xref:develop:connect/components/about.adoc[Redpanda Connect components reference]. +[source,yaml] +---- +include::ai-agents:example$weather_service.yaml[] +---- === YAML configuration rules Each YAML configuration (tool) should contain exactly one component type. The component type is inferred from the type you choose in the dropdown when creating or editing the MCP server. Valid component types are: -* `input` (for data sources) -* `output` (for data sinks) -* `processor` (for data transformations) -* `cache` (for caching intermediate results) +* xref:develop:connect/components/inputs/about.adoc[`input`] (for data sources) +* xref:develop:connect/components/outputs/about.adoc[`output`] (for data sinks) +* xref:develop:connect/components/processors/about.adoc[`processor`] (for data transformations and data access) +* xref:develop:connect/components/caches/about.adoc[`cache`] (for caching intermediate results) -.Correct example for inputs +.Correct example [source,yaml] ---- -label: my_input +label: event-reader redpanda: seed_brokers: [ "${REDPANDA_BROKERS}" ] topics: [ "events" ] @@ -127,9 +144,10 @@ meta: description: "Consume events from Redpanda" ---- -.Correct example for processors +.Correct example [source,yaml] ---- +label: fetch-example-data processors: - label: safe_operation try: @@ -151,7 +169,7 @@ processors: .Incorrect (do not include the input wrapper) [source,yaml] ---- -label: my_input +label: incorrect-example input: redpanda: seed_brokers: [ "${REDPANDA_BROKERS}" ] @@ -161,6 +179,7 @@ input: .Incorrect (multiple component types in one file) [source,yaml] ---- +label: incorrect-example input: redpanda: { ... } processors: @@ -169,730 +188,6 @@ output: redpanda: { ... } ---- -.Incorrect (try/catch as single processor) -[source,yaml] ----- -processors: - - label: operation - try: - - http: { ... } - catch: - - mutation: { ... } ----- - -=== Data generation and simple requests - -Start here if you're building your first MCP tools. These patterns cover data generation, HTTP API calls, and basic database queries. - -==== Data generators (input patterns) - -Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. - -This example generates a realistic user event message: - -[source,yaml] ----- -label: generate_input -generate: - mapping: | - let event_type = ["login", "logout", "purchase", "view_page", "click_button"].index(random_int(max:4)) - root = { - "id": uuid_v4(), - "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), - "user_id": random_int(min:1, max:10000), - "event_type": $event_type, - "data": { - "session_id": ksuid(), - "ip_address": "192.168.%v.%v".format(random_int(max:255), random_int(min:1, max:254)), - "user_agent": ["Chrome", "Firefox", "Safari", "Edge"].index(random_int(max:3)), - "amount": if $event_type == "purchase" { random_int(min:10, max:500) } else { null } - } - } - -meta: - mcp: - enabled: true - description: "Generate an example user event message with realistic data" - properties: [] ----- - -See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] - -==== External API calls (processor patterns) - -Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. - -[source,yaml] ----- -label: http_processor -processors: - - label: prepare_parameters - mutation: | - meta city_name = this.city_name - - label: fetch_weather - http: - url: 'https://wttr.in/${! @city_name }?format=j1' - verb: GET - headers: - Accept: "application/json" - User-Agent: "redpanda-mcp-server/1.0" - - label: format_response - mutation: | - root = { - "city": @city_name, - "temperature": this.current_condition.0.temp_C.number(), - "feels_like": this.current_condition.0.FeelsLikeC.number(), - "humidity": this.current_condition.0.humidity.number(), - "pressure": this.current_condition.0.pressure.number(), - "description": this.current_condition.0.weatherDesc.0.value, - "wind_speed": this.current_condition.0.windspeedKmph.number(), - "metadata": { - "source": "wttr.in", - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") - } - } - -meta: - mcp: - enabled: true - description: "Fetch current weather information for a specified city" - properties: - - name: city_name - type: string - description: "Name of the city to get weather information for" - required: true ----- - -See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] - -==== Database queries (processor patterns) - -Query external databases and return structured results. This pattern is essential for tools that need to access business data. - -[source,yaml] ----- -label: gcp_bigquery_select_processor -processors: - - label: prepare_parameters - mutation: | - meta customer_id = this.customer_id.string().catch("12345") - meta limit = this.limit.number().catch(10) - - label: query_bigquery - gcp_bigquery_select: - project: my-gcp-project - credentials_json: | - ${secrets.BIGQUERY_CREDENTIALS} - table: my_dataset.customer_orders - columns: - - "order_id" - - "customer_id" - - "order_date" - - "total_amount" - - "status" - where: customer_id = ? AND order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) - suffix: "ORDER BY order_date DESC LIMIT ?" - args_mapping: root = [ @customer_id, @limit ] - - label: format_response - mutation: | - root = { - "orders": this, - "metadata": { - "source": "BigQuery", - "customer_id": @customer_id, - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") - } - } - -meta: - mcp: - enabled: true - description: "Query customer orders from BigQuery" - properties: - - name: customer_id - type: string - description: "Customer ID to filter orders" - required: true - - name: limit - type: number - description: "Maximum number of orders to return" - required: false ----- - -See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) - -=== Redpanda integration and data publishing - -Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. - -==== Data publishers (output patterns) - -Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems or build caching systems. - -This example publishes a message to a Redpanda topic: - -[source,yaml] ----- -label: redpanda_output -redpanda: - seed_brokers: - - ${REDPANDA_BROKERS} - topic: ${! this.topic_name.string().catch("default-topic") } - timeout: 30s - tls: - enabled: true - sasl: - - mechanism: SCRAM-SHA-256 - username: ${secrets.REDPANDA_USERNAME} - password: ${secrets.REDPANDA_PASSWORD} -meta: - mcp: - enabled: true - description: Publishes a message to a specified Redpanda topic - properties: - - name: message - type: string - description: The message content to publish - required: true - - name: topic_name - type: string - description: The Redpanda topic to publish to - required: true ----- - -See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] - -==== Caching systems (cache patterns) - -Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. - -[source,yaml] ----- -label: redpanda_cache -redpanda: - seed_brokers: ["${REDPANDA_BROKERS}"] - topic: "mcp-cache-topic" - tls: - enabled: true - sasl: - - mechanism: "SCRAM-SHA-512" - username: "${secrets.MCP_REDPANDA_CREDENTIALS.username}" - password: "${secrets.MCP_REDPANDA_CREDENTIALS.password}" - -meta: - mcp: - enabled: true - description: "Redpanda-backed distributed cache using Kafka topics for persistence" ----- - -This example implements an in-memory cache for low-latency access to small datasets: - -[source,yaml] ----- -label: memory_cache -memory: - default_ttl: "5m" - init_values: - "user:1001": '{"name": "Alice", "role": "admin"}' - "user:1002": '{"name": "Bob", "role": "user"}' - "config:theme": "dark" - "config:language": "en" - shards: 4 - -meta: - mcp: - enabled: true - description: "In-memory cache for storing user data, configuration, and temporary values" ----- - -See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] - -=== Production workflows and observability - -Build enterprise-grade tools with error handling, validation, multi-step workflows, and monitoring. - -==== Parameter validation and type coercion - -Always validate and coerce input parameters to ensure your tools are robust: - -[source,yaml] ----- -processors: - - label: validate_params - mutation: | - # Validate required parameters - root = if !this.exists("user_id") { - throw("user_id parameter is required") - } else { this } - - # Type coercion with validation - meta user_id = this.user_id.string() - meta limit = this.limit.number().catch(10) - meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) ----- - -==== Dynamic configuration - -Build tools that adapt their behavior based on input parameters: - -[source,yaml] ----- -processors: - - label: dynamic_config - mutation: | - # Choose data source based on environment - meta env = this.environment | "production" - meta table_name = match @env { - "dev" => "dev_orders", - "staging" => "staging_orders", - "production" => "prod_orders", - _ => "dev_orders" - } - - # Adjust query complexity based on urgency - meta columns = if this.detailed.bool().catch(false) { - ["order_id", "customer_id", "total", "items", "shipping_address"] - } else { - ["order_id", "customer_id", "total"] - } ----- - -==== Error handling and fallbacks - -Implement error handling to make your tools reliable: - -[source,yaml] ----- -processors: - - label: primary_fetch - try: - - http: - url: "https://api.primary.com/data" - timeout: "10s" - catch: - - log: - message: "Primary API failed, trying fallback" - - label: fallback_fetch - http: - url: "https://api.fallback.com/data" - timeout: "15s" - - mutation: | - root.metadata.source = "fallback" - root.metadata.warning = "Primary source unavailable" ----- - -==== Conditional processing - -Build tools that branch based on input or data characteristics: - -[source,yaml] ----- -processors: - - label: conditional_processing - switch: - - check: this.data_type == "json" - processors: - - json: - operator: "parse" - - mutation: 'root.parsed_data = this' - - check: this.data_type == "csv" - processors: - - csv: - parse: true - - mutation: 'root.parsed_data = this' - - processors: - - mutation: 'root.error = "Unsupported data type"' ----- - -[[secrets]] -==== Secrets and credentials - -Securely handle multiple credentials and API keys. - -Here is an example of using an API key secret. - -. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. - -. Reference the secret in your pipeline configuration: -+ -[source,yaml] ----- -processors: - - label: call_external_api - http: - url: "https://api.example.com/data" - verb: GET - headers: - Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> - Accept: "application/json" ----- -+ -<1> The secret is injected at runtime. Never store the actual API key in your YAML. The actual secret value never appears in your configuration files or logs. - -==== Monitoring, debugging, and observability - -Use structured logging, request tracing, and performance metrics to gain insights into tool execution. - -[source,yaml] ----- -label: observable_tool -processors: - - label: init_tracing - mutation: | - # Generate correlation ID for request tracing - meta req_id = uuid_v7() - meta start_time = now() - - # Log request start with structured data - root.trace = { - "request_id": @req_id, - "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"), - "tool": "observable_tool", - "version": "1.0.0" - } - - - label: log_request_start - log: - message: "MCP tool request started" - fields: - request_id: "${! @req_id }" - tool_name: "observable_tool" - input_params: "${! this.without(\"trace\") }" - user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }" - level: "INFO" - - - label: finalize_response - mutation: | - # Calculate total execution time - meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000 - - # Add trace information to response - root.metadata = { - "request_id": @req_id, - "execution_time_ms": @duration, - "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), - "tool": "observable_tool", - "success": !this.exists("error") - } - - - label: log_completion - log: - message: "MCP tool request completed" - fields: - request_id: "${! @req_id }" - duration_ms: "${! this.metadata.execution_time_ms }" - success: "${! this.metadata.success }" - result_size: "${! content().length() }" - level: "INFO" - -meta: - mcp: - enabled: true - description: "Example tool with comprehensive observability and error handling" - properties: - - name: user_id - type: string - description: "User ID to fetch data for" - required: true ----- - -Observability features: - -* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing -* *Execution timing*: Track how long your tools take to execute using nanosecond precision -* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` -* *Request/response metadata*: Log input parameters and response characteristics -* *Success tracking*: Monitor whether operations complete successfully - -You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: - -[source,json] ----- -{ - "metadata": { - "execution_time_ms": 0.158977, - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "success": true, - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool" - }, - "trace": { - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool", - "version": "1.0.0" - }, - "user_id": "1" -} ----- - -For more advanced observability, use Redpanda Connect's GCP Cloud Trace integration to track request flows across your MCP tools: - -[source,yaml] ----- -# Configure tracer resource for GCP Cloud Trace -tracer_resources: - - label: gcp_tracer - gcp_cloudtrace: - project: "${secrets.GCP_PROJECT_ID}" - sampling_ratio: 1.0 - tags: - environment: "production" - cluster: "${CLUSTER_ID}" - service: "mcp-tools" - flush_interval: "5s" - -label: traced_tool -processors: - - label: start_span - tracer: - resource: gcp_tracer - operation_name: "mcp_tool_execution" - tags: - tool.name: "user_data_fetcher" - tool.version: "1.2.0" - user.id: "${! this.user_id }" - - - label: fetch_user_data - http: - url: "https://api.example.com/users/${! this.user_id }" - verb: GET - headers: - Authorization: "Bearer ${secrets.API_TOKEN}" - timeout: "10s" - - - label: add_span_details - tracer: - resource: gcp_tracer - tags: - http.status_code: "${! meta(\"http_status_code\") }" - response.size: "${! content().length() }" - api.endpoint: "users" - success: "${! !error().exists() }" - -meta: - mcp: - enabled: true - description: "User data fetcher with GCP Cloud Trace integration" - properties: - - name: user_id - type: string - required: true ----- - -Tracer benefits for MCP tools: - -* *Request flow visualization*: See how requests flow through your MCP tools and external services -* *Performance bottleneck identification*: Identify slow API calls or processing steps -* *Cross-service correlation*: Trace requests that span multiple services and systems -* *Error attribution*: Quickly identify which component caused a failure -* *Dependency mapping*: Understand relationships between your MCP tools and external services - -See also: xref:develop:connect/components/tracers/gcp_cloudtrace.adoc[`gcp_cloudtrace` tracer], xref:develop:connect/components/processors/log.adoc[`log` processor], xref:develop:connect/components/processors/try.adoc[`try` processor], xref:develop:connect/guides/bloblang/functions.adoc[Bloblang functions] (for timing and ID generation) - -==== Multi-step data enrichment - -Build tools that combine data from multiple sources. - -This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. - -[source,yaml] ----- -label: customer_enrichment -processors: - - label: fetch_customer_base - sql_select: - driver: "postgres" - dsn: "${secrets.POSTGRES_DSN}" - table: "customers" - where: "customer_id = ?" - args_mapping: 'root = [this.customer_id]' - - - label: enrich_with_orders - sql_select: - driver: "postgres" - dsn: "${secrets.POSTGRES_DSN}" - table: "orders" - where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" - args_mapping: 'root = [this.customer_id]' - - - label: combine_data - mutation: | - root = { - "customer": this.customers.index(0), - "recent_orders": this.orders, - "metrics": { - "total_orders": this.orders.length(), - "total_spent": this.orders.map_each(o -> o.total).sum(), - "avg_order_value": this.orders.map_each(o -> o.total).mean() - } - } - -meta: - mcp: - enabled: true - description: "Get comprehensive customer profile with recent order history and metrics" - properties: - - name: customer_id - type: string - description: "Customer ID to analyze" - required: true ----- - -See also: xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) - -==== Workflow orchestration - -Coordinate complex workflows with multiple steps and conditional logic. - -This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. - -[source,yaml] ----- -label: order_workflow -processors: - - label: validate_order - mutation: | - # Validation logic - root = if this.total <= 0 { - throw("Invalid order total") - } else { this } - - - label: mock_inventory_check - mutation: | - # Mock inventory data for testing - let inventory = { - "widget-001": {"quantity": 100, "name": "Standard Widget"}, - "widget-premium": {"quantity": 25, "name": "Premium Widget"}, - "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"} - } - - let product = $inventory.get(this.product_id) - root = if $product == null { - throw("Product not found: " + this.product_id) - } else if $product.quantity < this.quantity { - throw("Insufficient inventory. Available: " + $product.quantity.string()) - } else { - this.merge({ - "inventory_check": "passed", - "available_quantity": $product.quantity, - "product_name": $product.name - }) - } - - - label: route_by_priority - switch: - - check: 'this.total > 1000' - processors: - - label: mock_high_value_processing - mutation: | - # Mock premium processing - root = this.merge({ - "processing_tier": "premium", - "processing_time_estimate": "2-4 hours", - "assigned_rep": "premium-team@company.com", - "priority_score": 95 - }) - - - check: 'this.customer_tier == "vip"' - processors: - - label: mock_vip_processing - mutation: | - # Mock VIP processing - root = this.merge({ - "processing_tier": "vip", - "processing_time_estimate": "1-2 hours", - "assigned_rep": "vip-team@company.com", - "priority_score": 90, - "perks": ["expedited_shipping", "white_glove_service"] - }) - - - processors: - - label: mock_standard_processing - mutation: | - # Mock standard processing - root = this.merge({ - "processing_tier": "standard", - "processing_time_estimate": "24-48 hours", - "assigned_rep": "support@company.com", - "priority_score": 50 - }) - - - label: finalize_order - mutation: | - # Add final processing metadata - # Calculate estimated fulfillment by parsing processing time - let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number() - - root = this.merge({ - "order_status": "processed", - "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"), - "estimated_fulfillment": "TBD - calculated based on processing tier", - "processing_time_hours": $max_hours - }) - -meta: - mcp: - enabled: true - description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)" - properties: - - name: order_id - type: string - description: "Unique order identifier" - required: true - - name: product_id - type: string - description: "Product ID (try: widget-001, widget-premium, widget-limited)" - required: true - - name: quantity - type: number - description: "Quantity to order" - required: true - - name: total - type: number - description: "Order total in dollars" - required: true - - name: customer_tier - type: string - description: "Customer tier (optional: vip, standard)" - required: false ----- - -For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: - -[source,json] ----- -{ - "assigned_rep": "vip-team@company.com", - "available_quantity": 100, - "customer_tier": "vip", - "estimated_fulfillment": "TBD - calculated based on processing tier", - "inventory_check": "passed", - "order_id": "ORD001", - "order_status": "processed", - "perks": [ - "expedited_shipping", - "white_glove_service" - ], - "priority_score": 90, - "processed_at": "2025-09-16T09:05:29.138Z", - "processing_tier": "vip", - "processing_time_estimate": "1-2 hours", - "processing_time_hours": 2, - "product_id": "widget-001", - "product_name": "Standard Widget", - "quantity": 5, - "total": 250 -} ----- - -Notice how the workflow: - -. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. -. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. -. Routes by customer tier: Since `customer_tier` is "vip", it gets VIP processing with special `perks` and priority. -. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. -. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. - [[lint]] == Deploy and test your MCP server @@ -965,8 +260,10 @@ rpk cloud mcp proxy \ --install --client claude-code ---- + +-- <1> The target Redpanda Cloud cluster ID where your remote MCP server is hosted. <2> The unique ID of your deployed remote MCP server +-- + TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. + diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index 98af3e5e5..dcb12586a 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -101,7 +101,17 @@ rpk cloud mcp proxy --install --client claude-code --cluster-id --m - **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the remote MCP server. -== Next Steps +== MCP specification support + +Remote MCP server implements the open MCP protocol for tool exposure. Only the tool concept from the MCP server specification is supported. Features such as MCP resources and prompts are not yet available. + +For full details, see the official MCP server specification: link:https://modelcontextprotocol.io/specification/2025-06-18/server[modelcontextprotocol.io/specification/2025-06-18/server^]. + +== Get started * xref:ai-agents:mcp/remote/quickstart.adoc[] + +== Suggested reading + * xref:ai-agents:mcp/remote/developer-guide.adoc[] +* Learn more about MCP in the link:https://docs.anthropic.com/en/docs/mcp[official documentation^]. diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc new file mode 100644 index 000000000..26cb535bd --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc @@ -0,0 +1,306 @@ += Pipeline Patterns for Remote MCP Servers +:description: Catalog of pipeline patterns for remote MCP server tools in Redpanda Cloud. +:page-beta: true + +This page provides a reference catalog of pipeline patterns designed for use with remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. + +Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. + +For a high-level overview of MCP servers, see xref:ai-agents:mcp/overview.adoc[]. + +== Data generators + +Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. + +This example generates a realistic user event message: + +[source,yaml] +---- +include::ai-agents:example$generate_input.yaml[] +---- + +See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] + +== External API calls + +Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. + +[source,yaml] +---- +include::ai-agents:example$http_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] + +== Database queries + +Query external databases and return structured results. This pattern is essential for tools that need to access business data. + +[source,yaml] +---- +include::ai-agents:example$gcp_bigquery_select_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) + +=== Redpanda integration and data publishing + +Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. + +=== Data publishers (output patterns) + +Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems or build caching systems. + +This example publishes a message to a Redpanda topic: + +[source,yaml] +---- +include::ai-agents:example$redpanda_output.yaml[] +---- + +See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +=== Caching systems + +Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. + +[source,yaml] +---- +include::ai-agents:example$redpanda_cache.yaml[] +---- + +This example implements an in-memory cache for low-latency access to small datasets: + +[source,yaml] +---- +include::ai-agents:example$memory_cache.yaml[] +---- + +See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +== Production workflows and observability + +Build enterprise-grade tools with error handling, validation, multi-step workflows, and monitoring. + +=== Parameter validation and type coercion + +Always validate and coerce input parameters to ensure your tools are robust: + +[source,yaml] +---- +processors: + - label: validate_params + mutation: | + # Validate required parameters + root = if !this.exists("user_id") { + throw("user_id parameter is required") + } else { this } + + # Type coercion with validation + meta user_id = this.user_id.string() + meta limit = this.limit.number().catch(10) + meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) +---- + +=== Dynamic configuration + +Build tools that adapt their behavior based on input parameters: + +[source,yaml] +---- +processors: + - label: dynamic_config + mutation: | + # Choose data source based on environment + meta env = this.environment | "production" + meta table_name = match @env { + "dev" => "dev_orders", + "staging" => "staging_orders", + "production" => "prod_orders", + _ => "dev_orders" + } + + # Adjust query complexity based on urgency + meta columns = if this.detailed.bool().catch(false) { + ["order_id", "customer_id", "total", "items", "shipping_address"] + } else { + ["order_id", "customer_id", "total"] + } +---- + +=== Error handling and fallbacks + +Implement error handling to make your tools reliable: + +[source,yaml] +---- +processors: + - label: primary_fetch + try: + - http: + url: "https://api.primary.com/data" + timeout: "10s" + catch: + - log: + message: "Primary API failed, trying fallback" + - label: fallback_fetch + http: + url: "https://api.fallback.com/data" + timeout: "15s" + - mutation: | + root.metadata.source = "fallback" + root.metadata.warning = "Primary source unavailable" +---- + +=== Conditional processing + +Build tools that branch based on input or data characteristics: + +[source,yaml] +---- +processors: + - label: conditional_processing + switch: + - check: this.data_type == "json" + processors: + - json: + operator: "parse" + - mutation: 'root.parsed_data = this' + - check: this.data_type == "csv" + processors: + - csv: + parse: true + - mutation: 'root.parsed_data = this' + - processors: + - mutation: 'root.error = "Unsupported data type"' +---- + +[[secrets]] +=== Secrets and credentials + +Securely handle multiple credentials and API keys. + +Here is an example of using an API key secret. + +. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. + +. Reference the secret in your pipeline configuration: ++ +[source,yaml] +---- +processors: + - label: call_external_api + http: + url: "https://api.example.com/data" + verb: GET + headers: + Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> + Accept: "application/json" +---- ++ +<1> The secret is injected at runtime. Never store the actual API key in your YAML. The actual secret value never appears in your configuration files or logs. + +=== Monitoring, debugging, and observability + +Use structured logging, request tracing, and performance metrics to gain insights into tool execution. + +[source,yaml] +---- +include::ai-agents:example$observable_tool.yaml[] +---- + +Observability features: + +* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing +* *Execution timing*: Track how long your tools take to execute using nanosecond precision +* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` +* *Request/response metadata*: Log input parameters and response characteristics +* *Success tracking*: Monitor whether operations complete successfully + +You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: + +[source,json] +---- +{ + "metadata": { + "execution_time_ms": 0.158977, + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "success": true, + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool" + }, + "trace": { + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool", + "version": "1.0.0" + }, + "user_id": "1" +} +---- + +=== Multi-step data enrichment + +Build tools that combine data from multiple sources. + +This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. + +[source,yaml] +---- +include::ai-agents:example$customer_enrichment.yaml[] +---- + +See also: xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) + +=== Workflow orchestration + +Coordinate complex workflows with multiple steps and conditional logic. + +This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. + +[source,yaml] +---- +include::ai-agents:example$order_workflow.yaml[] +---- + +For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: + +[source,json] +---- +{ + "assigned_rep": "vip-team@company.com", + "available_quantity": 100, + "customer_tier": "vip", + "estimated_fulfillment": "TBD - calculated based on processing tier", + "inventory_check": "passed", + "order_id": "ORD001", + "order_status": "processed", + "perks": [ + "expedited_shipping", + "white_glove_service" + ], + "priority_score": 90, + "processed_at": "2025-09-16T09:05:29.138Z", + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "processing_time_hours": 2, + "product_id": "widget-001", + "product_name": "Standard Widget", + "quantity": 5, + "total": 250 +} +---- + +Notice how the workflow: + +. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. +. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. +. Routes by customer tier: Since `customer_tier` is "vip", it gets VIP processing with special `perks` and priority. +. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. +. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. + +== Suggested reading + +* xref:develop:connect/components/about.adoc[Components overview] +* xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] +* xref:develop:connect/configuration/secret-management.adoc[Secret management] \ No newline at end of file From 84463d24ead38b1ca5b4719b1a52e59ed71b721e Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 22 Sep 2025 12:49:10 +0100 Subject: [PATCH 17/30] Clarify mcp client --- .../ai-agents/pages/mcp/remote/developer-guide.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 889bb14df..61ed4ed9d 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -34,7 +34,7 @@ Secrets:: Credentials and tokens must be stored in the Secrets Store and referen . <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. . <>: Build an MCP tool using Redpanda Connect pipelines. . <>: Validate your pipeline, catch schema errors, then deploy and test with the MCP Inspector. -. <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. +. <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. . <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. [[access]] @@ -225,9 +225,9 @@ From the MCP Servers list, you can: See xref:ai-agents:mcp/remote/admin-guide.adoc[]. [[test]] -== Authenticate and connect your AI client +== Authenticate and connect your MCP client -To connect your local AI client to your remote MCP server: +To connect your local MCP client to your remote MCP server: . First, authenticate to Redpanda Cloud: + @@ -267,11 +267,11 @@ rpk cloud mcp proxy \ + TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. + -The proxy acts as a bridge between your local AI client and the remote MCP server running in your cluster. It: +The proxy acts as a bridge between your local client and the remote MCP server running in your cluster. It: + * Connects to your remote MCP server using your authentication token * Discovers and registers all tools from the remote server locally -* Proxies MCP requests from your AI client to the remote server +* Proxies MCP requests from your client to the remote server * Handles authentication by injecting your token into each request . Restart your client and invoke your tool. From a47a9e85e36e8421654b68df7dafea3f76810987 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Fri, 3 Oct 2025 13:47:26 +0100 Subject: [PATCH 18/30] Pipeline -> configuation --- .../pages/mcp/remote/developer-guide.adoc | 25 +++++++++---------- .../ai-agents/pages/mcp/remote/overview.adoc | 4 +-- .../pages/mcp/remote/pipeline-patterns.adoc | 10 ++++---- .../pages/mcp/remote/quickstart.adoc | 14 +++++++---- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 61ed4ed9d..249fa7787 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -19,21 +19,21 @@ You should also be familiar with YAML, HTTP APIs, and basic event-stream process == Concepts and architecture -Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write tool logic as Redpanda Connect pipelines and annotate them with MCP metadata so clients can discover and invoke them. +Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write each tool's logic using Redpanda Connect configuration and annotate them with MCP metadata so clients can discover and invoke them. -Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your remote MCP servers are implemented as Redpanda Connect pipelines that run inside your Redpanda Cloud cluster. +Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your remote MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. -Tool:: A single request/response operation exposed to MCP. Each tool is implemented as a Redpanda Connect pipeline and described to MCP with `meta.mcp`. +Tool:: A single request/response operation exposed to MCP. -Secrets:: Credentials and tokens must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). +Secrets:: Credentials and tokens that you want to use in your tools. These values must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). == Development workflow . <>: Select a Redpanda Cloud cluster to host your MCP server. . <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. . <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. -. <>: Build an MCP tool using Redpanda Connect pipelines. -. <>: Validate your pipeline, catch schema errors, then deploy and test with the MCP Inspector. +. <>: Build the logic for your MCP tool using Redpanda Connect configurations. +. <>: Validate your configuration, catch schema errors, then deploy and test with the MCP Inspector. . <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. . <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. @@ -81,27 +81,26 @@ meta: Property guidance: + * Use `string`, `number`, or `boolean` types. -* Validate ranges and enums using xref:guides:bloblang/about.adoc[Bloblang] inside the pipeline. * Mark only mandatory fields as required. -* Document defaults in the `description` and enforce them in the pipeline. +* Document defaults in the `description` and enforce them in the logic. -After defining your tool contract, implement the pipeline logic to handle input validation, defaults, and the main processing steps. +After defining your tool contract, implement the logic to handle input validation, defaults, and the main processing steps. [[provision-secrets]] == Provision secrets All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Follow these best practices: -- Reference secrets as `${secrets.NAME}` in your pipeline YAML. +- Reference secrets as `${secrets.NAME}` in your YAML configuration. - Never commit secrets to Git or reference them directly inline in configuration files. - Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). - Rotate secrets in the Secrets Store as needed. - Only request the scopes/roles required by your tool (principle of least privilege). -See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[Pipeline patterns]. +See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[MCP patterns]. [[pipeline-patterns]] -== Implement the pipeline +== Implement the logic of your MCP tool Use Redpanda Connect components to implement the logic of your MCP tool. Here are some best practices: @@ -191,7 +190,7 @@ output: [[lint]] == Deploy and test your MCP server -When your pipeline is ready, deploy it as a remote MCP server in Redpanda Cloud and test it with the built-in MCP Inspector. +When your tool is ready, deploy it as part of your remote MCP server in Redpanda Cloud and test it with the built-in MCP Inspector. === Deploy your MCP server diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index dcb12586a..095811cc3 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -14,7 +14,7 @@ Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that * *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. * *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. -* *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in pipelines. +* *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in your YAML configuration. * *Fast iteration*: Define tools as YAML configuration, deploy your MCP server, and your AI agents can use the new logic instantly. == Use cases @@ -46,7 +46,7 @@ Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that == How it works -. Build your tools as Redpanda Connect pipelines with `meta.mcp` metadata (description, properties, and logic). +. Build your tools as Redpanda Connect configurations with `meta.mcp` metadata (description, properties, and logic). . Deploy to your cluster. Redpanda Cloud hosts your MCP server with secure access. . Connect your AI client. AI assistants authenticate, discover available tools, and invoke them using natural language prompts. diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc index 26cb535bd..2a9b17b15 100644 --- a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc +++ b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc @@ -1,8 +1,8 @@ -= Pipeline Patterns for Remote MCP Servers -:description: Catalog of pipeline patterns for remote MCP server tools in Redpanda Cloud. += Patterns for Remote MCP Servers +:description: Catalog of patterns for remote MCP server tools in Redpanda Cloud. :page-beta: true -This page provides a reference catalog of pipeline patterns designed for use with remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. +This page provides a reference catalog of patterns designed for use with remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. @@ -184,7 +184,7 @@ Here is an example of using an API key secret. . Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. -. Reference the secret in your pipeline configuration: +. Reference the secret in your YAML configuration: + [source,yaml] ---- @@ -198,7 +198,7 @@ processors: Accept: "application/json" ---- + -<1> The secret is injected at runtime. Never store the actual API key in your YAML. The actual secret value never appears in your configuration files or logs. +<1> The secret is injected at runtime. Never store the actual API key in your YAML configuration. The actual secret value never appears in your configuration files or logs. === Monitoring, debugging, and observability diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 80470faa6..2779c31bf 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -8,7 +8,7 @@ include::ai-agents:partial$beta.adoc[] == Prerequisites -* A Redpanda Cloud cluster and subscription with *Remote MCP* enabled +* A Redpanda Cloud cluster with *Remote MCP* enabled * Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials (if needed) * At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] * link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed @@ -17,6 +17,8 @@ TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-gui == Create a remote MCP Server in Redpanda Cloud +. Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. + . Navigate to *Remote MCP*. + This page shows a list of existing servers. @@ -34,7 +36,7 @@ This page shows a list of existing servers. . Paste the following example YAML into the editor. This tool fetches Bitcoin price history from the CryptoCompare API. It accepts two optional parameters: `days` (number of days of history, default 30) and `currency` (target currency, default USD). + -.Reveal pipeline YAML +.Reveal MCP configuration [%collapsible] ==== [,yaml] @@ -113,7 +115,7 @@ meta: description: "Target currency for price data (default: USD)" required: false ---- -<1> **label:** Names this processor config for identification. +<1> **label:** Names your MCP tool for identification by clients. <2> **prepare_parameters:** Sets up parameters (`days`, `currency`) with defaults and builds the API endpoint URL. <3> **fetch_data:** Makes an HTTP GET request to the CryptoCompare API using the prepared endpoint. <4> **format_response:** Formats the API response into a structured JSON object, handling both success and error cases. @@ -129,7 +131,7 @@ It may take a few seconds to start. the status changes from *Starting* to *Runni == Connect a client -You can connect any MCP-compatible AI client to your remote MCP server. This example uses Claude Code. +You can connect any MCP-compatible client to your remote MCP server. This example uses Claude Code. The Redpanda CLI (`rpk`) acts as a local proxy, forwarding requests from your AI client to your own remote MCP server running in the Redpanda Cloud cluster. @@ -142,7 +144,9 @@ rpk cloud login + This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. -. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. It looks like this for BYOC and Dedicated clusters: +. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. ++ +For BYOC and Dedicated clusters, use: + [,bash] ---- From 0b5c52406cec6ed237264d055379bfc34dad2b73 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 11:28:47 +0100 Subject: [PATCH 19/30] Use templates for quickstart --- .../pages/mcp/remote/quickstart.adoc | 203 ++++++++---------- 1 file changed, 94 insertions(+), 109 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 2779c31bf..45fed1314 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -1,20 +1,56 @@ = Remote MCP Server Quickstart :page-beta: true -:description: Create and host a managed MCP server inside your Redpanda Cloud cluster. +:description: Create and host a managed MCP server inside your Redpanda Cloud cluster with tools for generating and publishing user event data. include::ai-agents:partial$beta.adoc[] -{description} +This quickstart guide shows you how to create and host a managed MCP server inside your Redpanda Cloud cluster. The server includes two tools: one for generating user event data and another for publishing that data to a Redpanda topic. == Prerequisites * A Redpanda Cloud cluster with *Remote MCP* enabled -* Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials (if needed) -* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] +* Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials. +* At least version 25.2.5 of the xref:manage:rpk/rpk-install.adoc[Redpanda CLI (`rpk`)] installed on your computer * link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. +== Prepare your cluster + +Before creating the MCP server, you need to set up a topic and user for event publishing. Use the Redpanda CLI to perform these steps. + +. Log in to your Redpanda Cloud account: ++ +[,bash] +---- +rpk cloud login +---- ++ +This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. + +. Create a topic called `events` for storing user event data: ++ +[,bash] +---- +rpk topic create events --partitions 3 --replicas 3 +---- + +. Create a user called `mcp` with a strong password: ++ +[,bash] +---- +rpk acl user create mcp --password +---- ++ +Save the password securely. You need it later when configuring the MCP server. + +. Grant the `mcp` user permissions to produce and consume from the `events` topic: ++ +[,bash] +---- +rpk acl create --allow-principal User:mcp --operation all --topic events +---- + == Create a remote MCP Server in Redpanda Cloud . Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. @@ -25,115 +61,45 @@ This page shows a list of existing servers. . Click *Create new MCP Server*. In *Metadata* add: + -* *Display Name*: A human-friendly name such as `crypto-inspector`. This name is shown in the Redpanda Cloud Console. It is not the name of the MCP server itself. -* *Description*: Explain what the server does. For example, `Fetches price history of cryptocurrencies from the CryptoCompare API`. -* *Tags*: Add key/value tags such as `owner=platform` or `env=prod`. +* *Display Name*: A human-friendly name such as `event-data-generator`. This name is shown in the Redpanda Cloud Console. It is not the name of the MCP server itself. +* *Description*: Explain what the server does. For example, `Generates fake user event data and publishes it to Redpanda topics`. +* *Tags*: Add key/value tags such as `owner=platform` or `env=demo`. * *Resources*: Choose a size (XSmall / Small / Medium / Large / XLarge). Larger sizes allow more concurrent requests and faster processing, but cost more. You can change this later. -. Click *Next* to define a tool. - -. Add the name of the tool, such as `btc_price_fetcher`. This is the name clients will use to call the tool. Make it descriptive so clients can discover it. - -. Paste the following example YAML into the editor. This tool fetches Bitcoin price history from the CryptoCompare API. It accepts two optional parameters: `days` (number of days of history, default 30) and `currency` (target currency, default USD). -+ -.Reveal MCP configuration -[%collapsible] -==== -[,yaml] ----- -label: btc_price_fetcher <1> - -processors: - - label: prepare_parameters <2> - mutation: | - meta days = this.days.catch(30) - meta currency = this.currency.catch("USD") - meta api_endpoint = "https://min-api.cryptocompare.com/data/v2/histoday?fsym=BTC&tsym=" + @currency + "&limit=" + (@days).string() - - - label: fetch_data <3> - http: - url: '${! @api_endpoint }' - verb: GET - headers: - Accept: "application/json" - User-Agent: "redpanda-mcp-server/1.0" - timeout: "30s" - retries: 3 - retry_period: "5s" - - - label: format_response <4> - mutation: | - root = if this.Response == "Success" { - { - "success": true, - "data": { - "symbol": "BTC", - "currency": @currency, - "period_days": @days.number(), - "total_records": this.Data.Data.length(), - "prices": if this.Data.Data != null { - this.Data.Data.map_each(entry -> { - "date": (entry.time.number() * 1000).ts_format("2006-01-02", "UTC"), - "timestamp": entry.time.number(), - "open": entry.open.number(), - "high": entry.high.number(), - "low": entry.low.number(), - "close": entry.close.number(), - "volume_from": entry.volumefrom.number(), - "volume_to": entry.volumeto.number() - }) - } else { [] } - }, - "metadata": { - "source": "CryptoCompare API", - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z", "UTC"), - "api_response": this.Response - } - } - } else { - { - "success": false, - "error": "API request failed: " + this.Response.format_json(), - "metadata": { - "source": "CryptoCompare API", - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z", "UTC") - } - } - } - -meta: - mcp: - enabled: true - description: "Fetches Bitcoin price history from CryptoCompare API for the specified number of days (default 30)" - properties: - - name: days - type: number - description: "Number of days of historical data to fetch (default: 30, max: 2000)" - required: false - - name: currency - type: string - description: "Target currency for price data (default: USD)" - required: false ----- -<1> **label:** Names your MCP tool for identification by clients. -<2> **prepare_parameters:** Sets up parameters (`days`, `currency`) with defaults and builds the API endpoint URL. -<3> **fetch_data:** Makes an HTTP GET request to the CryptoCompare API using the prepared endpoint. -<4> **format_response:** Formats the API response into a structured JSON object, handling both success and error cases. -==== +. Click *Next* to define tools. + +=== Add tools + +Tools define the actions your MCP server can perform. In this example, you create two tools: one for generating user event data and another for publishing that data to Redpanda. + +. From the *Template* dropdown, select *Generate Input*. ++ +The template populates the configuration with YAML for the tool definition. + +. Click *Add Tool* to create a second tool. + +. From the *Template* dropdown, select *Redpanda Output*. ++ +The template populates the configuration for publishing to Redpanda and a section for adding the required secrets is displayed. + +. Enter the values for the `mcp` user's credentials in the *Add Required Secrets* section. . Click *Lint* to check the configuration. You should see no errors. . Click *Create MCP Server* to deploy the server. + -It may take a few seconds to start. the status changes from *Starting* to *Running* when it's ready. +It may take a few seconds to start. The status changes from *Starting* to *Running* when it's ready. -. Open *MCP Inspector* tab to test the tool. Enter the request parameters such as `days=1` and `currency=USD`. Click *Run Tool*. You should see a response with Bitcoin price data. +. Open the *MCP Inspector* tab to test the tools: ++ +* For the `generate_input` tool, click *Run Tool* to generate sample event data. +* For the `redpanda_output` tool: Test with sample event data such as `user_id=user123`, `event_type=login`, and `timestamp=2025-10-21T10:30:00Z`. == Connect a client You can connect any MCP-compatible client to your remote MCP server. This example uses Claude Code. -The Redpanda CLI (`rpk`) acts as a local proxy, forwarding requests from your AI client to your own remote MCP server running in the Redpanda Cloud cluster. +The Redpanda CLI acts as a local proxy, forwarding requests from your AI client to your own remote MCP server running in the Redpanda Cloud cluster. . Log in to your Redpanda Cloud account: + @@ -173,10 +139,20 @@ rpk cloud mcp proxy \ claude ---- -. Ask Claude Code to run your tool. For example: +. Ask Claude Code to use your tools. For example: + -* "Fetch Bitcoin price history for the last 10 days in EUR." -* "Get BTC price data for 60 days in USD." +* "Generate 10 user events and then publish them to the events topic." +* "Create sample login events for users user001, user002, and user003, then publish them to Redpanda." +* "Generate purchase events with metadata and publish them to the events topic." + +You should see Claude Code calling your MCP server tools to generate and publish event data. You may need to respond to prompts for permissions to call the tools. + +When complete, you can verify the events were published by consuming from the `events` topic: + +[,bash] +---- +rpk topic consume events --num 10 +---- == Troubleshoot @@ -186,7 +162,7 @@ If your server is not starting or shows an error state: - Check the *Logs* tab for specific error messages. - Verify your YAML syntax by clicking *Lint*. -- Ensure all required secrets are created in the Secrets Store. +- Verify that the `mcp` user exists and has the correct permissions. === Connection issues @@ -197,13 +173,22 @@ If Claude Code can't connect to your server: - Ensure your server status shows *Running* in Redpanda Cloud. - Try restarting Claude Code after running the proxy command. -=== API request failures +=== Publishing failures -If the Bitcoin price tool returns errors: +If the publish tool returns errors: -- The CryptoCompare API may be rate limiting requests. Wait a minute and try again. -- Check your internet connection. -- Use the MCP Inspector in Redpanda Cloud to test with different parameters. +- Verify the `events` topic exists: ++ +```bash +rpk topic list +``` +- Check that the `mcp` user has the correct ACLs: ++ +```bash +rpk acl list +``` +- Use the MCP Inspector in Redpanda Cloud to test with different event data. +- Check the MCP server logs for authentication or authorization errors. == Next steps From 6139ca2a107446a58eeeb70dac8063b092651a33 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 11:28:55 +0100 Subject: [PATCH 20/30] Use templates for quickstart --- modules/ai-agents/pages/mcp/remote/quickstart.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 45fed1314..a7719a7b3 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -93,7 +93,7 @@ It may take a few seconds to start. The status changes from *Starting* to *Runni . Open the *MCP Inspector* tab to test the tools: + * For the `generate_input` tool, click *Run Tool* to generate sample event data. -* For the `redpanda_output` tool: Test with sample event data such as `user_id=user123`, `event_type=login`, and `timestamp=2025-10-21T10:30:00Z`. +* For the `redpanda_output` tool, enter some sample event data such as `user_id=user123`, `event_type=login`, and `timestamp=2025-10-21T10:30:00Z` then click *Run Tool* to publish it to the `events` topic. == Connect a client @@ -192,4 +192,4 @@ rpk acl list == Next steps -* xref:ai-agents:mcp/remote/developer-guide.adoc[] +Write your own tools to extend the MCP server functionality. For more information, see: xref:ai-agents:mcp/remote/developer-guide.adoc[] From 21c22ed917a3d7d98f437e18d5a3f50a24e4892e Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 11:51:49 +0100 Subject: [PATCH 21/30] Fix grammar --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 249fa7787..67b010716 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -48,7 +48,7 @@ Secrets:: Credentials and tokens that you want to use in your tools. These value + * *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). * *Description*: Brief description of what your server does. -* *Configuration*: This is where to put your YAML configuration to define the an MCP tool. +* *Configuration*: This is where to put your YAML configuration to define an MCP tool. [[contract]] == Design the tool contract and MCP metadata From 6c3af4296edbe22fd490cd25119f2c91b6d3082f Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 21 Oct 2025 13:45:00 +0100 Subject: [PATCH 22/30] Update modules/ai-agents/pages/mcp/remote/overview.adoc Co-authored-by: Paulo Borges --- modules/ai-agents/pages/mcp/remote/overview.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index 095811cc3..fd3607e56 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -1,6 +1,6 @@ = About Remote MCP Servers for Redpanda Cloud :page-beta: true -:description: Business value, use cases, and architecture for remote MCP servers in Redpanda Cloud. +:description: Discover how AI assistants can interact with your streaming data and how to connect them to Redpanda Cloud. include::ai-agents:partial$beta.adoc[] From 7d0b1a38ab3fe8591f03196b76e9e31009b679d7 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 21 Oct 2025 13:46:56 +0100 Subject: [PATCH 23/30] Apply suggestions from code review Co-authored-by: Paulo Borges --- modules/ai-agents/pages/index.adoc | 2 +- modules/ai-agents/pages/mcp/overview.adoc | 2 +- modules/ai-agents/pages/mcp/remote/index.adoc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc index 3f2a55af6..bf6b83d98 100644 --- a/modules/ai-agents/pages/index.adoc +++ b/modules/ai-agents/pages/index.adoc @@ -6,7 +6,7 @@ include::ai-agents:partial$beta.adoc[] -AI agents are configurable digital assistants that autonomously perform specialist tasks by leveraging large language models (LLMs) and connecting to external data sources and tools. +AI agents are configurable assistants that autonomously perform specialist tasks by leveraging large language models (LLMs) and connecting to external data sources and tools. Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 826e3c3df..9c397e639 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -8,7 +8,7 @@ include::ai-agents:partial$beta.adoc[] == Why use MCP with Redpanda Cloud? -With MCP, you can have natural conversations with your AI assistant about your streaming data infrastructure. Rather than remembering command-line syntax or navigating through web interfaces, you describe what you want in plain English and your AI assistant handles the technical details. +With MCP, you can have natural conversations with your AI assistant about your streaming data infrastructure. Rather than remembering command-line syntax or navigating through web interfaces, you describe what you want in plain language and your AI assistant handles the technical details. == MCP options in Redpanda Cloud diff --git a/modules/ai-agents/pages/mcp/remote/index.adoc b/modules/ai-agents/pages/mcp/remote/index.adoc index 1b7c481a3..43fa14dd4 100644 --- a/modules/ai-agents/pages/mcp/remote/index.adoc +++ b/modules/ai-agents/pages/mcp/remote/index.adoc @@ -1,4 +1,4 @@ = Remote MCP Servers for Redpanda Cloud :page-beta: true -:description: Find links to information about the Remote MCP servers for Redpanda Cloud and features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. +:description: Enable AI assistants to directly interact with your Redpanda Cloud clusters and streaming data. :page-layout: index From 6711dc6ca10dbb20719404e676cba3ee9038bd51 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 14:50:08 +0100 Subject: [PATCH 24/30] Add lint details --- .../pages/mcp/remote/developer-guide.adoc | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 67b010716..f847a0089 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -188,9 +188,25 @@ output: ---- [[lint]] -== Deploy and test your MCP server +== Lint and deploy your MCP server -When your tool is ready, deploy it as part of your remote MCP server in Redpanda Cloud and test it with the built-in MCP Inspector. +=== Lint your configuration + +Before deploying, validate your YAML configuration to catch syntax errors and schema violations: + +. In the Redpanda Cloud Console, after entering your YAML configuration, click *Lint*. +. The linter checks for: ++ +* Valid YAML syntax +* Correct Redpanda Connect component schemas +* Required MCP metadata fields +* Secret reference formatting +* Component configuration completeness + +. Fix any errors shown in the linting output before proceeding to deployment. +. A successful lint shows no errors and confirms your configuration is ready to deploy. + +TIP: Always lint your configuration after making changes. This catches issues early and prevents deployment failures. === Deploy your MCP server From 0922a059eebd49ea84907d43db28b670532e01cc Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 15:02:18 +0100 Subject: [PATCH 25/30] Update whats new --- modules/ai-agents/pages/mcp/overview.adoc | 8 +++--- .../pages/mcp/remote/admin-guide.adoc | 6 ++--- .../pages/mcp/remote/developer-guide.adoc | 26 +++++++++---------- .../ai-agents/pages/mcp/remote/overview.adoc | 4 +-- .../pages/mcp/remote/pipeline-patterns.adoc | 4 +-- .../pages/mcp/remote/quickstart.adoc | 6 ++--- .../get-started/pages/whats-new-cloud.adoc | 10 +++++++ 7 files changed, 37 insertions(+), 27 deletions(-) diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 9c397e639..2b2aab361 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -1,6 +1,6 @@ = MCP in Redpanda Cloud :page-beta: true -:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Local MCP server for Redpanda Cloud and remote MCP (managed). +:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Redpanda Cloud MCP Server and Remote MCP (managed). include::ai-agents:partial$beta.adoc[] @@ -16,7 +16,7 @@ In Redpanda Cloud, there are two complementary options to use MCP: [cols="1s,2a,2a"] |=== -| | Local MCP server for Redpanda Cloud | Remote MCP (managed) +| | Redpanda Cloud MCP Server | Remote MCP | Description | A server that gives your AI assistant access to your Redpanda Cloud account and clusters. @@ -41,9 +41,9 @@ TIP: The Redpanda documentation site also has a read-only MCP server that provid == Which should I use? -Use the **local MCP server for Redpanda Cloud** to operate your Redpanda Cloud account quickly from your laptop. +Use the **Redpanda Cloud MCP Server** to operate your Redpanda Cloud account quickly from your laptop. -Use the **remote MCP** to provide reusable tools for your team, running inside the cluster, authored with Redpanda Connect. +Use the **Remote MCP** to provide reusable tools for your team, running inside the cluster, authored with Redpanda Connect. == Get started diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc index de5821675..1c18c1ad6 100644 --- a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -1,10 +1,10 @@ = Manage Remote MCP Servers :page-beta: true -:description: Learn how to manage, edit, stop, resume, and delete remote MCP servers in Redpanda Cloud. +:description: Learn how to manage, edit, stop, resume, and delete Remote MCP servers in Redpanda Cloud. include::ai-agents:partial$beta.adoc[] -This guide is for Redpanda Cloud administrators who need to manage the lifecycle of remote MCP servers. +This guide is for Redpanda Cloud administrators who need to manage the lifecycle of Remote MCP servers. == Prerequisites @@ -12,7 +12,7 @@ You must have an existing MCP server. If you do not have one, see xref:ai-agents == Edit an existing MCP server -You can update the configuration, resources, or metadata of a remote MCP server at any time. +You can update the configuration, resources, or metadata of a Remote MCP server at any time. . In the Redpanda Cloud Console, navigate to *Remote MCP*. . Find the MCP server you want to edit and click its name. diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index f847a0089..32ee7378b 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -1,10 +1,10 @@ = Build Remote MCP Servers in Redpanda Cloud :page-beta: true -:description: Learn how to write and deploy remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. +:description: Learn how to write and deploy Remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. include::ai-agents:partial$beta.adoc[] -This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. +This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[Remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. == Prerequisites @@ -21,7 +21,7 @@ You should also be familiar with YAML, HTTP APIs, and basic event-stream process Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write each tool's logic using Redpanda Connect configuration and annotate them with MCP metadata so clients can discover and invoke them. -Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your remote MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. +Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your Remote MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. Tool:: A single request/response operation exposed to MCP. @@ -242,7 +242,7 @@ See xref:ai-agents:mcp/remote/admin-guide.adoc[]. [[test]] == Authenticate and connect your MCP client -To connect your local MCP client to your remote MCP server: +To connect your local MCP client to your Remote MCP server: . First, authenticate to Redpanda Cloud: + @@ -276,17 +276,17 @@ rpk cloud mcp proxy \ ---- + -- -<1> The target Redpanda Cloud cluster ID where your remote MCP server is hosted. -<2> The unique ID of your deployed remote MCP server +<1> The target Redpanda Cloud cluster ID where your Remote MCP server is hosted. +<2> The unique ID of your deployed Remote MCP server -- + TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. -+ -The proxy acts as a bridge between your local client and the remote MCP server running in your cluster. It: -+ -* Connects to your remote MCP server using your authentication token -* Discovers and registers all tools from the remote server locally -* Proxies MCP requests from your client to the remote server + +The proxy acts as a bridge between your local client and the Remote MCP server running in your cluster. It: + +* Connects to your Remote MCP server using your authentication token +* Discovers and registers all tools from the Remote server locally +* Proxies MCP requests from your client to the Remote server * Handles authentication by injecting your token into each request . Restart your client and invoke your tool. @@ -295,7 +295,7 @@ The proxy acts as a bridge between your local client and the remote MCP server r [NOTE] ==== -You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement of remote MCP. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. +You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement of Remote MCP. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. ==== :note-caption: Note diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index fd3607e56..f9981f225 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -10,7 +10,7 @@ Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that * Accelerate automation and insight by letting AI trigger real business actions. * Control exposure so that only the tools you tag are visible to AI. -== Why use a remote MCP server? +== Why use a Remote MCP server? * *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. * *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. @@ -99,7 +99,7 @@ rpk cloud login rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id ---- -- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the remote MCP server. +- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the Remote MCP server. == MCP specification support diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc index 2a9b17b15..70d589ece 100644 --- a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc +++ b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc @@ -1,8 +1,8 @@ = Patterns for Remote MCP Servers -:description: Catalog of patterns for remote MCP server tools in Redpanda Cloud. +:description: Catalog of patterns for Remote MCP server tools in Redpanda Cloud. :page-beta: true -This page provides a reference catalog of patterns designed for use with remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. +This page provides a reference catalog of patterns designed for use with Remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index a7719a7b3..158dcb197 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -51,7 +51,7 @@ Save the password securely. You need it later when configuring the MCP server. rpk acl create --allow-principal User:mcp --operation all --topic events ---- -== Create a remote MCP Server in Redpanda Cloud +== Create a Remote MCP Server in Redpanda Cloud . Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. @@ -97,9 +97,9 @@ It may take a few seconds to start. The status changes from *Starting* to *Runni == Connect a client -You can connect any MCP-compatible client to your remote MCP server. This example uses Claude Code. +You can connect any MCP-compatible client to your Remote MCP server. This example uses Claude Code. -The Redpanda CLI acts as a local proxy, forwarding requests from your AI client to your own remote MCP server running in the Redpanda Cloud cluster. +The Redpanda CLI acts as a local proxy, forwarding requests from your AI client to your own Remote MCP server running in the Redpanda Cloud cluster. . Log in to your Redpanda Cloud account: + diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index 15c1b5994..021a14730 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -6,6 +6,16 @@ This page lists new features added to Redpanda Cloud. +== October 2025 + +=== Remote MCP: beta + +Deploy managed MCP servers directly inside your Redpanda Cloud cluster with xref:ai-agents:mcp/remote/overview.adoc[Remote MCP]. Unlike the Redpanda Cloud MCP Server, Remote MCP servers run within your cluster and can process data streams, generate synthetic data, and publish directly to Redpanda topics. + +Create custom AI tools using templates or write your own Redpanda Connect configurations to build event-driven workflows. Remote MCP servers provide AI assistants with streaming data capabilities, enabling use cases like real-time data generation, stream processing, and event publishing. + +Get started with the xref:ai-agents:mcp/remote/quickstart.adoc[quickstart guide] or explore advanced patterns in the xref:ai-agents:mcp/remote/developer-guide.adoc[developer guide]. + == September 2025 === Redpanda Cloud MCP Server: beta From ca4cc7cae091d2b6b9038c97113cfa7850043036 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 15:03:54 +0100 Subject: [PATCH 26/30] Clarify docs MCP --- modules/ai-agents/pages/mcp/overview.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 2b2aab361..7fc29e0ac 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -37,7 +37,7 @@ In Redpanda Cloud, there are two complementary options to use MCP: - Generate a summary of user activity from the events stream. |=== -TIP: The Redpanda documentation site also has a read-only MCP server that provides access to Redpanda docs and examples. This server has no access to your Redpanda Cloud account or clusters. See xref:home:ROOT:mcp-setup.adoc[]. +TIP: The Redpanda documentation site includes a read-only MCP server that gives you access to Redpanda documentation and code examples through AI assistants like Claude Desktop. This server only provides access to public documentation content. It cannot access your Redpanda Cloud account, clusters, or any private data. This is useful for getting help with Redpanda concepts and examples without connecting your actual Redpanda environment. For setup instructions, see xref:home:ROOT:mcp-setup.adoc[]. == Which should I use? From 976b1f2f071fb34e2f9f6157bcb8cc503af48798 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 16:03:00 +0100 Subject: [PATCH 27/30] Document property restrictions --- .../pages/mcp/remote/developer-guide.adoc | 62 +++++++++++++++---- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 32ee7378b..22e533b6d 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -55,34 +55,43 @@ Secrets:: Credentials and tokens that you want to use in your tools. These value Each MCP tool must declare its interface using `meta.mcp` metadata. This metadata allows AI clients to discover and invoke the tool correctly. -Define a clear, stable interface for each tool. Keep the description task-oriented and keep parameters to a minimum. +Define a clear, stable interface for each tool. Keep the description task-oriented and keep properties to a minimum. [source,yaml] ---- meta: mcp: enabled: true <1> - description: "Fetches a compact summary from an external API using two optional parameters." <2> + description: "Publishes a message to a specified Redpanda topic" <2> properties: <3> - - name: parameter1 + - name: message type: string - description: "Primary filter; defaults to provider standard when omitted." - required: false - - name: parameter2 - type: number - description: "Limit of results (1-100)." - required: false + description: "The message content to publish" + required: true + - name: topic_name + type: string + description: "The Redpanda topic to publish to" + required: true ---- <1> Set `meta.mcp.enabled: true` to expose the tool using MCP. <2> A concise description that explains what the tool does. This should be understandable by an AI model. -<3> List the input parameters (properties) for the tool. +<3> Define the properties that AI clients can provide when calling this tool. + +Properties define the inputs that AI clients can provide when calling your tool. Each property has: + +* **name**: The parameter name AI clients will use +* **type**: Data type (`string`, `number`, or `boolean`) +* **description**: Human-readable explanation of what this property does +* **required**: Whether the property is mandatory (`true`) or optional (`false`) + Property guidance: + -* Use `string`, `number`, or `boolean` types. -* Mark only mandatory fields as required. -* Document defaults in the `description` and enforce them in the logic. +* Use descriptive names that clearly indicate the property's purpose. +* Provide clear descriptions that explain both the purpose and any constraints. +* Mark only mandatory fields as required. Prefer optional properties with sensible defaults. +* Document default values in the description and implement them in your tool logic. +* Consider component type restrictions: `input` components only support a `count` property, `cache` components have no custom properties, and `output` components receive properties as arrays for batch operations. After defining your tool contract, implement the logic to handle input validation, defaults, and the main processing steps. @@ -128,6 +137,33 @@ Each YAML configuration (tool) should contain exactly one component type. The co * xref:develop:connect/components/processors/about.adoc[`processor`] (for data transformations and data access) * xref:develop:connect/components/caches/about.adoc[`cache`] (for caching intermediate results) +=== Property restrictions by component type + +Different component types have different property capabilities when exposed as MCP tools: + +[cols="1,2,2"] +|=== +| Component Type | Property Support | Details + +| `input` +| Only supports `count` property +| AI clients can specify how many messages to read, but you cannot define custom properties. + +| `cache` +| No custom properties +| Properties are hardcoded to `key` and `value` for cache operations. + +| `output` +| Custom properties supported +| AI sees properties as an array for batch operations: `[{prop1, prop2}, {prop1, prop2}]`. + +| `processor` +| Custom properties supported +| You can define any properties needed for data processing operations. +|=== + +=== Configuration examples + .Correct example [source,yaml] ---- From 26c2dd62f45cf5e104b1dfd39bea087f70540495 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 17:02:01 +0100 Subject: [PATCH 28/30] Remove private beta --- modules/ai-agents/pages/mcp/overview.adoc | 1 - modules/ai-agents/pages/mcp/remote/admin-guide.adoc | 1 - modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 1 - modules/ai-agents/pages/mcp/remote/overview.adoc | 1 - modules/ai-agents/pages/mcp/remote/quickstart.adoc | 1 - modules/ai-agents/partials/beta.adoc | 1 - modules/get-started/pages/cloud-overview.adoc | 2 +- 7 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 modules/ai-agents/partials/beta.adoc diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 7fc29e0ac..6024bfeae 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -2,7 +2,6 @@ :page-beta: true :description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Redpanda Cloud MCP Server and Remote MCP (managed). -include::ai-agents:partial$beta.adoc[] **Model Context Protocol (MCP)** is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc index 1c18c1ad6..ec09bc792 100644 --- a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -2,7 +2,6 @@ :page-beta: true :description: Learn how to manage, edit, stop, resume, and delete Remote MCP servers in Redpanda Cloud. -include::ai-agents:partial$beta.adoc[] This guide is for Redpanda Cloud administrators who need to manage the lifecycle of Remote MCP servers. diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index 22e533b6d..cab8d0e4f 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -2,7 +2,6 @@ :page-beta: true :description: Learn how to write and deploy Remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. -include::ai-agents:partial$beta.adoc[] This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[Remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index f9981f225..ee5fc6dd7 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -2,7 +2,6 @@ :page-beta: true :description: Discover how AI assistants can interact with your streaming data and how to connect them to Redpanda Cloud. -include::ai-agents:partial$beta.adoc[] Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that you can write and run inside your Redpanda Cloud cluster. They let you expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 158dcb197..dd213173d 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -2,7 +2,6 @@ :page-beta: true :description: Create and host a managed MCP server inside your Redpanda Cloud cluster with tools for generating and publishing user event data. -include::ai-agents:partial$beta.adoc[] This quickstart guide shows you how to create and host a managed MCP server inside your Redpanda Cloud cluster. The server includes two tools: one for generating user event data and another for publishing that data to a Redpanda topic. diff --git a/modules/ai-agents/partials/beta.adoc b/modules/ai-agents/partials/beta.adoc deleted file mode 100644 index cf01d3ce5..000000000 --- a/modules/ai-agents/partials/beta.adoc +++ /dev/null @@ -1 +0,0 @@ -NOTE: Remote MCP servers are in **private beta** and are free to use until they are released as Generally Available (GA). \ No newline at end of file diff --git a/modules/get-started/pages/cloud-overview.adoc b/modules/get-started/pages/cloud-overview.adoc index 7f33c5c16..7691345ae 100644 --- a/modules/get-started/pages/cloud-overview.adoc +++ b/modules/get-started/pages/cloud-overview.adoc @@ -387,7 +387,7 @@ The following features are currently in beta in Redpanda Cloud: * BYOVNet for Azure * Secrets Management for BYOVPC on GCP and AWS * Remote Read Replicas for AWS and GCP -* Redpanda Cloud MCP Server +* xref:ai-agents:index.adoc[AI agents with MCP servers] * Several Redpanda Connect components == Next steps From 41dd1aa47a80b954fd4c7779ec86d2404afa149f Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 17:07:11 +0100 Subject: [PATCH 29/30] Remove private beta --- modules/ai-agents/pages/index.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc index bf6b83d98..5c32a437f 100644 --- a/modules/ai-agents/pages/index.adoc +++ b/modules/ai-agents/pages/index.adoc @@ -4,8 +4,6 @@ :page-layout: index :page-aliases: develop:agents/about.adoc, develop:ai-agents/about.adoc -include::ai-agents:partial$beta.adoc[] - AI agents are configurable assistants that autonomously perform specialist tasks by leveraging large language models (LLMs) and connecting to external data sources and tools. Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: From 41de2d09ea4c2bc1864243c92fa3a3e757adf8c3 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 21 Oct 2025 17:19:26 +0100 Subject: [PATCH 30/30] Fix formatting --- modules/ai-agents/pages/mcp/remote/developer-guide.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc index cab8d0e4f..5fc7affaf 100644 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -85,7 +85,7 @@ Properties define the inputs that AI clients can provide when calling your tool. * **required**: Whether the property is mandatory (`true`) or optional (`false`) Property guidance: -+ + * Use descriptive names that clearly indicate the property's purpose. * Provide clear descriptions that explain both the purpose and any constraints. * Mark only mandatory fields as required. Prefer optional properties with sensible defaults.