diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index fb1da4145..31f02be74 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -69,10 +69,18 @@ ** xref:security:secrets.adoc[Secrets] ** xref:security:cloud-safety-reliability.adoc[Safety and Reliability] -* xref:ai-agents:mcp/local/index.adoc[] -** xref:ai-agents:mcp/local/overview.adoc[Overview] -** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] -** xref:ai-agents:mcp/local/configuration.adoc[Configure] +* xref:ai-agents:index.adoc[AI Agents] +** xref:ai-agents:mcp/overview.adoc[MCP Overview] +** xref:ai-agents:mcp/local/index.adoc[Local Redpanda Cloud MCP] +*** xref:ai-agents:mcp/local/overview.adoc[Overview] +*** xref:ai-agents:mcp/local/quickstart.adoc[Quickstart] +*** xref:ai-agents:mcp/local/configuration.adoc[Configure] +** xref:ai-agents:mcp/remote/index.adoc[Remote MCP] +*** xref:ai-agents:mcp/remote/overview.adoc[Overview] +*** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] +*** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] +*** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] +*** xref:ai-agents:mcp/remote/pipeline-patterns.adoc[MCP Server Patterns] * xref:develop:connect/about.adoc[Redpanda Connect] ** xref:develop:connect/connect-quickstart.adoc[Quickstart] @@ -490,8 +498,9 @@ ***** xref:reference:rpk/rpk-cloud/rpk-cloud-cluster-select.adoc[] **** xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[] **** xref:reference:rpk/rpk-cloud/rpk-cloud-logout.adoc[] -**** rpk cloud mcp +**** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp.adoc[] ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[] +***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[] ***** xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[] *** xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[] **** xref:reference:rpk/rpk-cluster/rpk-cluster-config.adoc[] diff --git a/modules/ai-agents/examples/customer_enrichment.yaml b/modules/ai-agents/examples/customer_enrichment.yaml new file mode 100644 index 000000000..209624818 --- /dev/null +++ b/modules/ai-agents/examples/customer_enrichment.yaml @@ -0,0 +1,39 @@ +label: customer_enrichment +processors: + - label: fetch_customer_base + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "customers" + where: "customer_id = ?" + args_mapping: 'root = [this.customer_id]' + + - label: enrich_with_orders + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "orders" + where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" + args_mapping: 'root = [this.customer_id]' + + - label: combine_data + mutation: | + root = { + "customer": this.customers.index(0), + "recent_orders": this.orders, + "metrics": { + "total_orders": this.orders.length(), + "total_spent": this.orders.map_each(o -> o.total).sum(), + "avg_order_value": this.orders.map_each(o -> o.total).mean() + } + } + +meta: + mcp: + enabled: true + description: "Get comprehensive customer profile with recent order history and metrics" + properties: + - name: customer_id + type: string + description: "Customer ID to analyze" + required: true \ No newline at end of file diff --git a/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml b/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml new file mode 100644 index 000000000..c6620580d --- /dev/null +++ b/modules/ai-agents/examples/gcp_bigquery_select_processor.yaml @@ -0,0 +1,45 @@ +label: gcp_bigquery_select_processor +processors: + - label: prepare_parameters + mutation: | + meta customer_id = this.customer_id.string().catch("12345") + meta limit = this.limit.number().catch(10) + - label: query_bigquery + gcp_bigquery_select: + project: my-gcp-project + credentials_json: | + ${secrets.BIGQUERY_CREDENTIALS} + table: my_dataset.customer_orders + columns: + - "order_id" + - "customer_id" + - "order_date" + - "total_amount" + - "status" + where: customer_id = ? AND order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) + suffix: "ORDER BY order_date DESC LIMIT ?" + args_mapping: root = [ @customer_id, @limit ] + - label: format_response + mutation: | + root = { + "orders": this, + "metadata": { + "source": "BigQuery", + "customer_id": @customer_id, + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Query customer orders from BigQuery" + properties: + - name: customer_id + type: string + description: "Customer ID to filter orders" + required: true + - name: limit + type: number + description: "Maximum number of orders to return" + required: false diff --git a/modules/ai-agents/examples/generate_input.yaml b/modules/ai-agents/examples/generate_input.yaml new file mode 100644 index 000000000..0c8106144 --- /dev/null +++ b/modules/ai-agents/examples/generate_input.yaml @@ -0,0 +1,22 @@ +label: generate_input +generate: + mapping: | + let event_type = ["login", "logout", "purchase", "view_page", "click_button"].index(random_int(max:4)) + root = { + "id": uuid_v4(), + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "user_id": random_int(min:1, max:10000), + "event_type": $event_type, + "data": { + "session_id": ksuid(), + "ip_address": "192.168.%v.%v".format(random_int(max:255), random_int(min:1, max:254)), + "user_agent": ["Chrome", "Firefox", "Safari", "Edge"].index(random_int(max:3)), + "amount": if $event_type == "purchase" { random_int(min:10, max:500) } else { null } + } + } + +meta: + mcp: + enabled: true + description: "Generate an example user event message with realistic data" + properties: [] diff --git a/modules/ai-agents/examples/http_processor.yaml b/modules/ai-agents/examples/http_processor.yaml new file mode 100644 index 000000000..6e523a1b3 --- /dev/null +++ b/modules/ai-agents/examples/http_processor.yaml @@ -0,0 +1,37 @@ +label: fetch-weather +processors: + - label: prepare_parameters + mutation: | + meta city_name = this.city_name + - label: fetch_weather + http: + url: 'https://wttr.in/${! @city_name }?format=j1' + verb: GET + headers: + Accept: "application/json" + User-Agent: "redpanda-mcp-server/1.0" + - label: format_response + mutation: | + root = { + "city": @city_name, + "temperature": this.current_condition.0.temp_C.number(), + "feels_like": this.current_condition.0.FeelsLikeC.number(), + "humidity": this.current_condition.0.humidity.number(), + "pressure": this.current_condition.0.pressure.number(), + "description": this.current_condition.0.weatherDesc.0.value, + "wind_speed": this.current_condition.0.windspeedKmph.number(), + "metadata": { + "source": "wttr.in", + "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") + } + } + +meta: + mcp: + enabled: true + description: "Fetch current weather information for a specified city" + properties: + - name: city_name + type: string + description: "Name of the city to get weather information for" + required: true diff --git a/modules/ai-agents/examples/memory_cache.yaml b/modules/ai-agents/examples/memory_cache.yaml new file mode 100644 index 000000000..bf67ab8ce --- /dev/null +++ b/modules/ai-agents/examples/memory_cache.yaml @@ -0,0 +1,14 @@ +label: memory_cache +memory: + default_ttl: "5m" + init_values: + "user:1001": '{"name": "Alice", "role": "admin"}' + "user:1002": '{"name": "Bob", "role": "user"}' + "config:theme": "dark" + "config:language": "en" + shards: 4 + +meta: + mcp: + enabled: true + description: "In-memory cache for storing user data, configuration, and temporary values" diff --git a/modules/ai-agents/examples/observable_tool.yaml b/modules/ai-agents/examples/observable_tool.yaml new file mode 100644 index 000000000..e2596a796 --- /dev/null +++ b/modules/ai-agents/examples/observable_tool.yaml @@ -0,0 +1,60 @@ +label: observable_tool +processors: + - label: init_tracing + mutation: | + # Generate correlation ID for request tracing + meta req_id = uuid_v7() + meta start_time = now() + + # Log request start with structured data + root.trace = { + "request_id": @req_id, + "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "version": "1.0.0" + } + + - label: log_request_start + log: + message: "MCP tool request started" + fields: + request_id: "${! @req_id }" + tool_name: "observable_tool" + input_params: "${! this.without(\"trace\") }" + user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }" + level: "INFO" + + - label: finalize_response + mutation: | + # Calculate total execution time + meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000 + + # Add trace information to response + root.metadata = { + "request_id": @req_id, + "execution_time_ms": @duration, + "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), + "tool": "observable_tool", + "success": !this.exists("error") + } + + - label: log_completion + log: + message: "MCP tool request completed" + fields: + request_id: "${! @req_id }" + duration_ms: "${! this.metadata.execution_time_ms }" + success: "${! this.metadata.success }" + result_size: "${! content().length() }" + level: "INFO" + +meta: + tags: [ example ] + mcp: + enabled: true + description: "Example tool with comprehensive observability and error handling" + properties: + - name: user_id + type: string + description: "User ID to fetch data for" + required: true diff --git a/modules/ai-agents/examples/order_workflow.yaml b/modules/ai-agents/examples/order_workflow.yaml new file mode 100644 index 000000000..ad0dc29ee --- /dev/null +++ b/modules/ai-agents/examples/order_workflow.yaml @@ -0,0 +1,107 @@ +label: order_workflow +processors: + - label: validate_order + mutation: | + # Validation logic + root = if this.total <= 0 { + throw("Invalid order total") + } else { this } + + - label: mock_inventory_check + mutation: | + # Mock inventory data for testing + let inventory = { + "widget-001": {"quantity": 100, "name": "Standard Widget"}, + "widget-premium": {"quantity": 25, "name": "Premium Widget"}, + "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"} + } + + let product = $inventory.get(this.product_id) + root = if $product == null { + throw("Product not found: " + this.product_id) + } else if $product.quantity < this.quantity { + throw("Insufficient inventory. Available: " + $product.quantity.string()) + } else { + this.merge({ + "inventory_check": "passed", + "available_quantity": $product.quantity, + "product_name": $product.name + }) + } + + - label: route_by_priority + switch: + - check: 'this.total > 1000' + processors: + - label: mock_high_value_processing + mutation: | + # Mock premium processing + root = this.merge({ + "processing_tier": "premium", + "processing_time_estimate": "2-4 hours", + "assigned_rep": "premium-team@company.com", + "priority_score": 95 + }) + + - check: 'this.customer_tier == "vip"' + processors: + - label: mock_vip_processing + mutation: | + # Mock VIP processing + root = this.merge({ + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "assigned_rep": "vip-team@company.com", + "priority_score": 90, + "perks": ["expedited_shipping", "white_glove_service"] + }) + + - processors: + - label: mock_standard_processing + mutation: | + # Mock standard processing + root = this.merge({ + "processing_tier": "standard", + "processing_time_estimate": "24-48 hours", + "assigned_rep": "support@company.com", + "priority_score": 50 + }) + + - label: finalize_order + mutation: | + # Add final processing metadata + # Calculate estimated fulfillment by parsing processing time + let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number() + + root = this.merge({ + "order_status": "processed", + "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"), + "estimated_fulfillment": "TBD - calculated based on processing tier", + "processing_time_hours": $max_hours + }) + +meta: + mcp: + enabled: true + description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)" + properties: + - name: order_id + type: string + description: "Unique order identifier" + required: true + - name: product_id + type: string + description: "Product ID (try: widget-001, widget-premium, widget-limited)" + required: true + - name: quantity + type: number + description: "Quantity to order" + required: true + - name: total + type: number + description: "Order total in dollars" + required: true + - name: customer_tier + type: string + description: "Customer tier (optional: vip, standard)" + required: false \ No newline at end of file diff --git a/modules/ai-agents/examples/redpanda_cache.yaml b/modules/ai-agents/examples/redpanda_cache.yaml new file mode 100644 index 000000000..49be191d9 --- /dev/null +++ b/modules/ai-agents/examples/redpanda_cache.yaml @@ -0,0 +1,15 @@ +label: redpanda_cache +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] + topic: "mcp-cache-topic" + tls: + enabled: true + sasl: + - mechanism: "SCRAM-SHA-512" + username: "${secrets.MCP_REDPANDA_CREDENTIALS.username}" + password: "${secrets.MCP_REDPANDA_CREDENTIALS.password}" + +meta: + mcp: + enabled: true + description: "Redpanda-backed distributed cache using Kafka topics for persistence" diff --git a/modules/ai-agents/examples/redpanda_output.yaml b/modules/ai-agents/examples/redpanda_output.yaml new file mode 100644 index 000000000..324b135af --- /dev/null +++ b/modules/ai-agents/examples/redpanda_output.yaml @@ -0,0 +1,25 @@ +label: redpanda_output +redpanda: + seed_brokers: + - ${REDPANDA_BROKERS} + topic: ${! this.topic_name.string().catch("default-topic") } + timeout: 30s + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: ${secrets.REDPANDA_USERNAME} + password: ${secrets.REDPANDA_PASSWORD} +meta: + mcp: + enabled: true + description: Publishes a message to a specified Redpanda topic + properties: + - name: message + type: string + description: The message content to publish + required: true + - name: topic_name + type: string + description: The Redpanda topic to publish to + required: true diff --git a/modules/ai-agents/examples/weather_service.yaml b/modules/ai-agents/examples/weather_service.yaml new file mode 100644 index 000000000..1a6c56fe4 --- /dev/null +++ b/modules/ai-agents/examples/weather_service.yaml @@ -0,0 +1,60 @@ +label: weather-service +processors: + - label: validate_inputs + mutation: | + # Validate and sanitize city input + meta city = this.city.string(). + re_replace_all("[^a-zA-Z\\s\\-]", ""). + trim() + + # Check for empty input + root = if @city == "" { + throw("City name cannot be empty") + } else { "" } + + - label: fetch_weather_data + try: + - http: + url: "https://wttr.in/${! @city }?format=j1" + verb: GET + headers: + User-Agent: "redpanda-connect-mcp/1.0" + timeout: "10s" + - mutation: | + root = { + "city": @city, + "temperature_c": this.current_condition.0.temp_C.number(), + "temperature_f": this.current_condition.0.temp_F.number(), + "feels_like_c": this.current_condition.0.FeelsLikeC.number(), + "humidity": this.current_condition.0.humidity.number(), + "description": this.current_condition.0.weatherDesc.0.value, + "wind_speed_kmh": this.current_condition.0.windspeedKmph.number(), + "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") + } + - log: + message: "Weather data fetched for city: ${! @city }" + level: "INFO" + + - label: handle_weather_errors + catch: + - mutation: | + root = { + "error": "Failed to fetch weather data", + "city": @city, + "details": error(), + "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") + } + - log: + message: "Weather API error for city ${! @city }: ${! error() }" + level: "ERROR" + +meta: + tags: [ weather, example ] + mcp: + enabled: true + description: "Get current weather conditions for any city worldwide" + properties: + - name: city + type: string + description: "Name of the city (such as 'San Francisco', 'London')" + required: true diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc new file mode 100644 index 000000000..5c32a437f --- /dev/null +++ b/modules/ai-agents/pages/index.adoc @@ -0,0 +1,13 @@ += AI Agents in Redpanda Cloud +:page-beta: true +:description: Learn about AI agents and the tools Redpanda Cloud provides for building them. +:page-layout: index +:page-aliases: develop:agents/about.adoc, develop:ai-agents/about.adoc + +AI agents are configurable assistants that autonomously perform specialist tasks by leveraging large language models (LLMs) and connecting to external data sources and tools. + +Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: + +**Local MCP server for Redpanda Cloud**: A built-in server that gives your AI assistant direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. + +**Remote MCP (managed)**: Your own custom MCP servers built with Redpanda Connect pipelines and hosted inside your Redpanda Cloud cluster. These let you create reusable tools for your team that integrate with knowledge bases, APIs, and web content while running close to your data. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/configuration.adoc b/modules/ai-agents/pages/mcp/local/configuration.adoc index 60781047d..7b4453f3c 100644 --- a/modules/ai-agents/pages/mcp/local/configuration.adoc +++ b/modules/ai-agents/pages/mcp/local/configuration.adoc @@ -1,8 +1,8 @@ -= Configure Redpanda Cloud MCP += Configure the Local MCP server for Redpanda Cloud :page-beta: true -:description: Learn how to configure the Redpanda Cloud MCP Server for your AI assistant, including auto and manual client setup, enabling deletes, and security considerations. +:description: Learn how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. -This page explains how to configure the Redpanda Cloud MCP Server for your AI assistant, including auto and manual client setup, enabling deletes, and security considerations. +This page explains how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. == Prerequisites @@ -31,7 +31,7 @@ If you need to update the integration, re-run the install command for your clien == Manual configuration for other MCP clients -If you're using another MCP-compatible client, you can manually configure it to use Redpanda Cloud MCP Server. Follow these steps: +If you're using another MCP-compatible client, you can manually configure it to use the local MCP server for Redpanda Cloud. Follow these steps: Add an MCP server entry to your client's configuration (example shown in JSON). Adjust paths for your system. @@ -175,10 +175,6 @@ Expected response shapes (examples): + .. Stop the server with `Ctrl+C`. -== Common issues - -The server runs locally, but it must reach Redpanda Cloud endpoints to perform operations. - === Client can't find the MCP server * Re-run the install for your MCP client. diff --git a/modules/ai-agents/pages/mcp/local/index.adoc b/modules/ai-agents/pages/mcp/local/index.adoc index e70b573f8..16429590c 100644 --- a/modules/ai-agents/pages/mcp/local/index.adoc +++ b/modules/ai-agents/pages/mcp/local/index.adoc @@ -1,5 +1,4 @@ -= Redpanda Cloud MCP Server += Local MCP server for Redpanda Cloud :page-beta: true -:description: Find links to information about the Redpanda Cloud MCP Server and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. +:description: Find links to information about the Local MCP server for Redpanda Cloud and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. :page-layout: index -:page-aliases: develop:agents/about.adoc \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/overview.adoc b/modules/ai-agents/pages/mcp/local/overview.adoc index 8160dccc3..cc624f400 100644 --- a/modules/ai-agents/pages/mcp/local/overview.adoc +++ b/modules/ai-agents/pages/mcp/local/overview.adoc @@ -1,16 +1,16 @@ -= About the Redpanda Cloud MCP Server += About the Local MCP server for Redpanda Cloud :page-beta: true -:description: Learn about the Redpanda Cloud MCP Server, which lets AI assistants securely access and operate your Redpanda Cloud account and clusters. +:description: Learn about the local MCP server for Redpanda Cloud, which lets AI assistants securely access and operate your Redpanda Cloud account and clusters. -The Redpanda Cloud MCP Server lets AI assistants securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: +The local MCP server for Redpanda Cloud lets AI assistants securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: * link:https://docs.redpanda.com/api/doc/cloud-controlplane/[Control Plane] actions, such as creating a Redpanda Cloud cluster or listing clusters. * link:https://docs.redpanda.com/api/doc/cloud-dataplane/[Data Plane] actions, such as creating topics or listing topics. By speaking natural language to your assistant, you can ask it to perform Redpanda operations on your behalf. The MCP server runs locally on your machine and authenticates to Redpanda Cloud using a Redpanda Cloud token. -image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking Redpanda Cloud MCP to list topics in a cluster.] +image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking the local MCP server for Redpanda Cloud to list topics in a cluster.] == What you can do @@ -32,7 +32,7 @@ NOTE: The MCP server does **not** expose delete endpoints by default. You can en . Authenticate to Redpanda Cloud and receive a token using the xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command]. . Configure your MCP client using the xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[`rpk cloud mcp install`] command. Your client then starts the server on-demand using xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[`rpk cloud mcp stdio`], authenticating with the Redpanda Cloud token from `rpk cloud login`. -. Prompt your assistant to perform Redpanda operations. The Redpanda Cloud MCP Server executes them in your Redpanda Cloud account using your Redpanda Cloud token. +. Prompt your assistant to perform Redpanda operations. The local MCP server for Redpanda Cloud executes them in your Redpanda Cloud account using your Redpanda Cloud token. === Components @@ -40,6 +40,14 @@ NOTE: The MCP server does **not** expose delete endpoints by default. You can en * Redpanda CLI (`rpk`) for obtaining a token and starting the local MCP server. * Redpanda Cloud account that the local MCP server can connect to and issue API requests. +== Security considerations + +MCP servers authenticate to Redpanda Cloud using your personal or service account credentials. However, there is **no auditing or access control** that distinguishes between actions performed by MCP servers versus direct API calls: + +* All API actions appear in Redpanda Cloud's internal logs as coming from the authenticated user account, not the specific MCP server. +* You cannot audit which MCP server performed which operations, as Redpanda Cloud logs are not accessible to users. +* You cannot restrict specific MCP servers to only certain API endpoints or resources. + == Next steps * xref:ai-agents:mcp/local/quickstart.adoc[] diff --git a/modules/ai-agents/pages/mcp/local/quickstart.adoc b/modules/ai-agents/pages/mcp/local/quickstart.adoc index 55eddc00b..6cf7ee111 100644 --- a/modules/ai-agents/pages/mcp/local/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/local/quickstart.adoc @@ -1,14 +1,10 @@ -= Redpanda Cloud MCP Quickstart += Local MCP server for Redpanda Cloud Quickstart :page-beta: true -:description: Connect your Claude AI assistant to your Redpanda Cloud account and clusters using the Redpanda Cloud MCP Server. +:description: Connect your Claude AI assistant to your Redpanda Cloud account and clusters using the local MCP server for Redpanda Cloud. -In this quickstart, you'll get your Claude AI assistant talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[Redpanda Cloud MCP Server]. - -//// -To be used when Remote MCP is ready for public beta +In this quickstart, you'll get your Claude AI assistant talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud]. If you're trying to deploy your own MCP server as a managed service inside your cluster, see xref:ai-agents:mcp/remote/quickstart.adoc[]. -//// == Prerequisites diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc new file mode 100644 index 000000000..6024bfeae --- /dev/null +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -0,0 +1,50 @@ += MCP in Redpanda Cloud +:page-beta: true +:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Redpanda Cloud MCP Server and Remote MCP (managed). + + +**Model Context Protocol (MCP)** is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. + +== Why use MCP with Redpanda Cloud? + +With MCP, you can have natural conversations with your AI assistant about your streaming data infrastructure. Rather than remembering command-line syntax or navigating through web interfaces, you describe what you want in plain language and your AI assistant handles the technical details. + +== MCP options in Redpanda Cloud + +In Redpanda Cloud, there are two complementary options to use MCP: + +[cols="1s,2a,2a"] +|=== +| | Redpanda Cloud MCP Server | Remote MCP + +| Description +| A server that gives your AI assistant access to your Redpanda Cloud account and clusters. +| Your own MCP server built with Redpanda Connect and hosted inside your Redpanda Cloud cluster. + +| Where it runs +| Your computer +| In your Redpanda Cloud cluster (managed by Redpanda) + +| Who builds it +| Redpanda +| You, with Redpanda Connect + +| Example use cases (prompts you might ask) +| - Create a new cluster called `dev-analytics` for testing. +- List all topics in my production cluster. +| - Analyze the last 100 orders and show me the top product categories. +- Generate a summary of user activity from the events stream. +|=== + +TIP: The Redpanda documentation site includes a read-only MCP server that gives you access to Redpanda documentation and code examples through AI assistants like Claude Desktop. This server only provides access to public documentation content. It cannot access your Redpanda Cloud account, clusters, or any private data. This is useful for getting help with Redpanda concepts and examples without connecting your actual Redpanda environment. For setup instructions, see xref:home:ROOT:mcp-setup.adoc[]. + +== Which should I use? + +Use the **Redpanda Cloud MCP Server** to operate your Redpanda Cloud account quickly from your laptop. + +Use the **Remote MCP** to provide reusable tools for your team, running inside the cluster, authored with Redpanda Connect. + +== Get started + +* xref:ai-agents:mcp/local/quickstart.adoc[] +* xref:ai-agents:mcp/remote/quickstart.adoc[] diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc new file mode 100644 index 000000000..ec09bc792 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -0,0 +1,56 @@ += Manage Remote MCP Servers +:page-beta: true +:description: Learn how to manage, edit, stop, resume, and delete Remote MCP servers in Redpanda Cloud. + + +This guide is for Redpanda Cloud administrators who need to manage the lifecycle of Remote MCP servers. + +== Prerequisites + +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +== Edit an existing MCP server + +You can update the configuration, resources, or metadata of a Remote MCP server at any time. + +. In the Redpanda Cloud Console, navigate to *Remote MCP*. +. Find the MCP server you want to edit and click its name. +. Click *Edit configuration*. +. Make your changes. +. Click *Save* to apply changes. + +[NOTE] +==== +Editing a running MCP server may cause a brief interruption. Review changes before deploying to production. +==== + +== Stop (pause) an MCP server + +Pausing a server stops all tool execution and releases compute resources, but preserves configuration and state. + +. In the *Remote MCP* list, find the server you want to stop. +. Click the three dots and select *Stop*. +. Confirm the action. + +[NOTE] +==== +While stopped, the server does not respond to MCP requests. Start it again to restore service. +==== + +== Resume a stopped MCP server + +. In the *Remote MCP* list, find the stopped server. +. Click the three dots and select *Start*. +. Wait for the status to show *Running* before reconnecting clients. + +== Delete an MCP server + +Deleting a server is permanent and cannot be undone. + +. In the *Remote MCP* list, find the server to delete. +. Click the three dots and select *Delete*. +. Confirm the deletion when prompted. + +== API and automation + +All of these actions can also be performed using the Redpanda Cloud API version v1alpha3. See the link:https://github.com/redpanda-data/console/blob/master/proto/gen/openapi/openapi.v1alpha3.yaml[OpenAPI specification file] for endpoints to manage MCP servers. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc new file mode 100644 index 000000000..5fc7affaf --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc @@ -0,0 +1,349 @@ += Build Remote MCP Servers in Redpanda Cloud +:page-beta: true +:description: Learn how to write and deploy Remote MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. + + +This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[Remote MCP servers] that are managed in Redpanda Cloud. Remote MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. + +== Prerequisites + +* Access to a Redpanda Cloud cluster +* Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries +* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer] +* (Optional) An AI assistant like Claude or Claude Code for testing + +TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. + +You should also be familiar with YAML, HTTP APIs, and basic event-stream processing concepts. + +== Concepts and architecture + +Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write each tool's logic using Redpanda Connect configuration and annotate them with MCP metadata so clients can discover and invoke them. + +Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your Remote MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. + +Tool:: A single request/response operation exposed to MCP. + +Secrets:: Credentials and tokens that you want to use in your tools. These values must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). + +== Development workflow + +. <>: Select a Redpanda Cloud cluster to host your MCP server. +. <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. +. <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. +. <>: Build the logic for your MCP tool using Redpanda Connect configurations. +. <>: Validate your configuration, catch schema errors, then deploy and test with the MCP Inspector. +. <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. +. <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. + +[[access]] +== Access MCP Servers in Redpanda Cloud + +. Log into your link:https://cloud.redpanda.com[Redpanda Cloud Console^]. +. Select your cluster from the cluster list. +. In the left sidebar, navigate to *Remote MCP*. +. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. +. Fill in the required fields: ++ +* *Name*: A unique identifier for your server (for example `weather-tool`, `customer-lookup`). +* *Description*: Brief description of what your server does. +* *Configuration*: This is where to put your YAML configuration to define an MCP tool. + +[[contract]] +== Design the tool contract and MCP metadata + +Each MCP tool must declare its interface using `meta.mcp` metadata. This metadata allows AI clients to discover and invoke the tool correctly. + +Define a clear, stable interface for each tool. Keep the description task-oriented and keep properties to a minimum. + +[source,yaml] +---- +meta: + mcp: + enabled: true <1> + description: "Publishes a message to a specified Redpanda topic" <2> + properties: <3> + - name: message + type: string + description: "The message content to publish" + required: true + - name: topic_name + type: string + description: "The Redpanda topic to publish to" + required: true +---- + +<1> Set `meta.mcp.enabled: true` to expose the tool using MCP. +<2> A concise description that explains what the tool does. This should be understandable by an AI model. +<3> Define the properties that AI clients can provide when calling this tool. ++ +Properties define the inputs that AI clients can provide when calling your tool. Each property has: + +* **name**: The parameter name AI clients will use +* **type**: Data type (`string`, `number`, or `boolean`) +* **description**: Human-readable explanation of what this property does +* **required**: Whether the property is mandatory (`true`) or optional (`false`) + +Property guidance: + +* Use descriptive names that clearly indicate the property's purpose. +* Provide clear descriptions that explain both the purpose and any constraints. +* Mark only mandatory fields as required. Prefer optional properties with sensible defaults. +* Document default values in the description and implement them in your tool logic. +* Consider component type restrictions: `input` components only support a `count` property, `cache` components have no custom properties, and `output` components receive properties as arrays for batch operations. + +After defining your tool contract, implement the logic to handle input validation, defaults, and the main processing steps. + +[[provision-secrets]] +== Provision secrets + +All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Follow these best practices: + +- Reference secrets as `${secrets.NAME}` in your YAML configuration. +- Never commit secrets to Git or reference them directly inline in configuration files. +- Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). +- Rotate secrets in the Secrets Store as needed. +- Only request the scopes/roles required by your tool (principle of least privilege). + +See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[MCP patterns]. + +[[pipeline-patterns]] +== Implement the logic of your MCP tool + +Use Redpanda Connect components to implement the logic of your MCP tool. Here are some best practices: + +* **Single responsibility**: Each tool should do one thing well. +* **Descriptive naming**: Use clear, specific labels like `fetch-user-profile` instead of generic names like `get-data`. The top-level label becomes the tool's name in the MCP server. +* **Input validation**: Always validate and sanitize user inputs using xref:develop:connect/guides/bloblang/about.adoc[Bloblang]. +* **Error handling**: Provide meaningful error messages. +* **Documentation**: Write clear descriptions that explain what the tool does and what it returns. + +For common patterns and examples, see xref:ai-agents:mcp/remote/pipeline-patterns.adoc[]. + +Here's a complete example that demonstrates best practices: + +[source,yaml] +---- +include::ai-agents:example$weather_service.yaml[] +---- + +=== YAML configuration rules + +Each YAML configuration (tool) should contain exactly one component type. The component type is inferred from the type you choose in the dropdown when creating or editing the MCP server. Valid component types are: + +* xref:develop:connect/components/inputs/about.adoc[`input`] (for data sources) +* xref:develop:connect/components/outputs/about.adoc[`output`] (for data sinks) +* xref:develop:connect/components/processors/about.adoc[`processor`] (for data transformations and data access) +* xref:develop:connect/components/caches/about.adoc[`cache`] (for caching intermediate results) + +=== Property restrictions by component type + +Different component types have different property capabilities when exposed as MCP tools: + +[cols="1,2,2"] +|=== +| Component Type | Property Support | Details + +| `input` +| Only supports `count` property +| AI clients can specify how many messages to read, but you cannot define custom properties. + +| `cache` +| No custom properties +| Properties are hardcoded to `key` and `value` for cache operations. + +| `output` +| Custom properties supported +| AI sees properties as an array for batch operations: `[{prop1, prop2}, {prop1, prop2}]`. + +| `processor` +| Custom properties supported +| You can define any properties needed for data processing operations. +|=== + +=== Configuration examples + +.Correct example +[source,yaml] +---- +label: event-reader +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "events" ] + consumer_group: "mcp-reader" + +meta: + mcp: + enabled: true + description: "Consume events from Redpanda" +---- + +.Correct example +[source,yaml] +---- +label: fetch-example-data +processors: + - label: safe_operation + try: + - http: + url: "https://api.example.com/data" + timeout: "10s" + - mutation: | + root = this.merge({"processed": true}) + + - label: handle_errors + catch: + - mutation: | + root = { + "error": "Operation failed", + "details": error() + } +---- + +.Incorrect (do not include the input wrapper) +[source,yaml] +---- +label: incorrect-example +input: + redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "events" ] +---- + +.Incorrect (multiple component types in one file) +[source,yaml] +---- +label: incorrect-example +input: + redpanda: { ... } +processors: + - mutation: { ... } +output: + redpanda: { ... } +---- + +[[lint]] +== Lint and deploy your MCP server + +=== Lint your configuration + +Before deploying, validate your YAML configuration to catch syntax errors and schema violations: + +. In the Redpanda Cloud Console, after entering your YAML configuration, click *Lint*. +. The linter checks for: ++ +* Valid YAML syntax +* Correct Redpanda Connect component schemas +* Required MCP metadata fields +* Secret reference formatting +* Component configuration completeness + +. Fix any errors shown in the linting output before proceeding to deployment. +. A successful lint shows no errors and confirms your configuration is ready to deploy. + +TIP: Always lint your configuration after making changes. This catches issues early and prevents deployment failures. + +=== Deploy your MCP server + +. Click *Create MCP Server* to deploy the server. +. Monitor the *Status* column. It will show one of: ++ +* *Starting*: Server is being deployed. +* *Running*: Server is active and ready to receive requests. +* *Error*: Check logs for deployment issues. + +=== Test with MCP Inspector + +. When your server status shows *Running*, click on the server name. +. Navigate to the *MCP Inspector* tab. +. This built-in tool lets you: ++ +* View all available tools your server exposes. +* Test tool calls with sample parameters. +* See request/response data. +* Debug issues without setting up external clients. + +=== Manage your MCP server + +From the MCP Servers list, you can: + +* *View logs*: Click the server name, then go to the *Logs* tab to see execution logs and errors. +* *Update configuration*: Edit the YAML and redeploy. +* *Start/Stop*: Control server lifecycle. +* *Delete*: Remove the server and clean up resources. + +See xref:ai-agents:mcp/remote/admin-guide.adoc[]. + +[[test]] +== Authenticate and connect your MCP client + +To connect your local MCP client to your Remote MCP server: + +. First, authenticate to Redpanda Cloud: ++ +[source,bash] +---- +rpk cloud login +---- ++ +This opens a browser window for sign-in and stores your authentication token locally. + +. Install the MCP proxy for your client (Claude/Claude Code). ++ +For BYOC and Dedicated clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--cluster-id \ <1> +--mcp-server-id \ <2> +--install --client claude-code +---- ++ +For Serverless clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--serverless-cluster-id \ <1> +--mcp-server-id \ <2> +--install --client claude-code +---- ++ +-- +<1> The target Redpanda Cloud cluster ID where your Remote MCP server is hosted. +<2> The unique ID of your deployed Remote MCP server +-- ++ +TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. + +The proxy acts as a bridge between your local client and the Remote MCP server running in your cluster. It: + +* Connects to your Remote MCP server using your authentication token +* Discovers and registers all tools from the Remote server locally +* Proxies MCP requests from your client to the Remote server +* Handles authentication by injecting your token into each request + +. Restart your client and invoke your tool. + +:note-caption: Building your own agent? + +[NOTE] +==== +You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement of Remote MCP. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. +==== + +:note-caption: Note + +[[observe]] +== Observe and debug your MCP tools + +You can view execution logs in the Redpanda Cloud Console under your MCP server's *Logs* tab. + +== Suggested reading + +* xref:develop:connect/components/about.adoc[Redpanda Connect components reference] +* xref:develop:connect/guides/bloblang/about.adoc[Bloblang language guide] +* xref:develop:connect/configuration/secret-management.adoc[Secret management in Redpanda Connect] +* xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command reference] +* xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[`rpk cloud mcp proxy` command reference] diff --git a/modules/ai-agents/pages/mcp/remote/index.adoc b/modules/ai-agents/pages/mcp/remote/index.adoc new file mode 100644 index 000000000..43fa14dd4 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/index.adoc @@ -0,0 +1,4 @@ += Remote MCP Servers for Redpanda Cloud +:page-beta: true +:description: Enable AI assistants to directly interact with your Redpanda Cloud clusters and streaming data. +:page-layout: index diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc new file mode 100644 index 000000000..ee5fc6dd7 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -0,0 +1,116 @@ += About Remote MCP Servers for Redpanda Cloud +:page-beta: true +:description: Discover how AI assistants can interact with your streaming data and how to connect them to Redpanda Cloud. + + +Remote MCP (Model Context Protocol) servers are managed, hosted MCP servers that you can write and run inside your Redpanda Cloud cluster. They let you expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: + +* Make your data and workflows accessible to AI assistants and LLMs, safely and on your terms. +* Accelerate automation and insight by letting AI trigger real business actions. +* Control exposure so that only the tools you tag are visible to AI. + +== Why use a Remote MCP server? + +* *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. +* *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. +* *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in your YAML configuration. +* *Fast iteration*: Define tools as YAML configuration, deploy your MCP server, and your AI agents can use the new logic instantly. + +== Use cases + +[cols="1s,3a"] +|=== +|Category |Example prompts + +|Operational monitoring +|* Check partition lag for customer-events topic +* Show me the top 10 producers by message volume today +* Get schema registry health status + +|Data enrichment and analysis +|* Fetch user profile data and recent orders for customer ID 12345 +* Get real-time stock prices for symbols in my portfolio topic +* Analyze sentiment of latest product reviews + +|Team productivity +|* Deploy my microservice to the staging environment +* Generate load test data for the payments service +* Create a summary dashboard of this week's incident reports + +|Business intelligence +|* What are the trending products in the last 24 hours? +* Show revenue impact of the latest feature deployment +* Get customer satisfaction scores from support tickets +|=== + +== How it works + +. Build your tools as Redpanda Connect configurations with `meta.mcp` metadata (description, properties, and logic). +. Deploy to your cluster. Redpanda Cloud hosts your MCP server with secure access. +. Connect your AI client. AI assistants authenticate, discover available tools, and invoke them using natural language prompts. + +== Example: Customer analytics tool + +Here's how you might expose a customer analytics pipeline as an MCP tool: + +[source,yaml] +---- +label: analyze_customer_orders +processors: + - label: fetch_customer_data + sql_select: + driver: "postgres" + dsn: "${secrets.POSTGRES_DSN}" + table: "orders" + where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" + args_mapping: 'root = [this.customer_id]' + + - label: calculate_metrics + mutation: | + root = { + "customer_id": this.customer_id, + "total_orders": this.length(), + "total_spent": this.map_each(o -> o.total).sum(), + "avg_order_value": this.map_each(o -> o.total).mean(), + "last_order_date": this.map_each(o -> o.created_at).max() + } + +meta: + mcp: + enabled: true + description: "Analyze a customer's order history and spending patterns over the last 30 days" + properties: + - name: customer_id + type: string + description: "Customer ID to analyze" + required: true +---- + +This tool can be invoked by an AI assistant with prompts like "analyze the order history for customer ID 12345" without needing to know the underlying SQL or processing logic. + +== Authentication options + +- **MCP proxy (recommended):** For tools with built-in MCP clients like Claude Code: ++ +[,bash] +---- +rpk cloud login +rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id +---- + +- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the Remote MCP server. + +== MCP specification support + +Remote MCP server implements the open MCP protocol for tool exposure. Only the tool concept from the MCP server specification is supported. Features such as MCP resources and prompts are not yet available. + +For full details, see the official MCP server specification: link:https://modelcontextprotocol.io/specification/2025-06-18/server[modelcontextprotocol.io/specification/2025-06-18/server^]. + +== Get started + +* xref:ai-agents:mcp/remote/quickstart.adoc[] + +== Suggested reading + +* xref:ai-agents:mcp/remote/developer-guide.adoc[] +* Learn more about MCP in the link:https://docs.anthropic.com/en/docs/mcp[official documentation^]. diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc new file mode 100644 index 000000000..70d589ece --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc @@ -0,0 +1,306 @@ += Patterns for Remote MCP Servers +:description: Catalog of patterns for Remote MCP server tools in Redpanda Cloud. +:page-beta: true + +This page provides a reference catalog of patterns designed for use with Remote MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. + +Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. + +For a high-level overview of MCP servers, see xref:ai-agents:mcp/overview.adoc[]. + +== Data generators + +Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. + +This example generates a realistic user event message: + +[source,yaml] +---- +include::ai-agents:example$generate_input.yaml[] +---- + +See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] + +== External API calls + +Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. + +[source,yaml] +---- +include::ai-agents:example$http_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] + +== Database queries + +Query external databases and return structured results. This pattern is essential for tools that need to access business data. + +[source,yaml] +---- +include::ai-agents:example$gcp_bigquery_select_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) + +=== Redpanda integration and data publishing + +Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. + +=== Data publishers (output patterns) + +Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems or build caching systems. + +This example publishes a message to a Redpanda topic: + +[source,yaml] +---- +include::ai-agents:example$redpanda_output.yaml[] +---- + +See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +=== Caching systems + +Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. + +[source,yaml] +---- +include::ai-agents:example$redpanda_cache.yaml[] +---- + +This example implements an in-memory cache for low-latency access to small datasets: + +[source,yaml] +---- +include::ai-agents:example$memory_cache.yaml[] +---- + +See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +== Production workflows and observability + +Build enterprise-grade tools with error handling, validation, multi-step workflows, and monitoring. + +=== Parameter validation and type coercion + +Always validate and coerce input parameters to ensure your tools are robust: + +[source,yaml] +---- +processors: + - label: validate_params + mutation: | + # Validate required parameters + root = if !this.exists("user_id") { + throw("user_id parameter is required") + } else { this } + + # Type coercion with validation + meta user_id = this.user_id.string() + meta limit = this.limit.number().catch(10) + meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) +---- + +=== Dynamic configuration + +Build tools that adapt their behavior based on input parameters: + +[source,yaml] +---- +processors: + - label: dynamic_config + mutation: | + # Choose data source based on environment + meta env = this.environment | "production" + meta table_name = match @env { + "dev" => "dev_orders", + "staging" => "staging_orders", + "production" => "prod_orders", + _ => "dev_orders" + } + + # Adjust query complexity based on urgency + meta columns = if this.detailed.bool().catch(false) { + ["order_id", "customer_id", "total", "items", "shipping_address"] + } else { + ["order_id", "customer_id", "total"] + } +---- + +=== Error handling and fallbacks + +Implement error handling to make your tools reliable: + +[source,yaml] +---- +processors: + - label: primary_fetch + try: + - http: + url: "https://api.primary.com/data" + timeout: "10s" + catch: + - log: + message: "Primary API failed, trying fallback" + - label: fallback_fetch + http: + url: "https://api.fallback.com/data" + timeout: "15s" + - mutation: | + root.metadata.source = "fallback" + root.metadata.warning = "Primary source unavailable" +---- + +=== Conditional processing + +Build tools that branch based on input or data characteristics: + +[source,yaml] +---- +processors: + - label: conditional_processing + switch: + - check: this.data_type == "json" + processors: + - json: + operator: "parse" + - mutation: 'root.parsed_data = this' + - check: this.data_type == "csv" + processors: + - csv: + parse: true + - mutation: 'root.parsed_data = this' + - processors: + - mutation: 'root.error = "Unsupported data type"' +---- + +[[secrets]] +=== Secrets and credentials + +Securely handle multiple credentials and API keys. + +Here is an example of using an API key secret. + +. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. + +. Reference the secret in your YAML configuration: ++ +[source,yaml] +---- +processors: + - label: call_external_api + http: + url: "https://api.example.com/data" + verb: GET + headers: + Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> + Accept: "application/json" +---- ++ +<1> The secret is injected at runtime. Never store the actual API key in your YAML configuration. The actual secret value never appears in your configuration files or logs. + +=== Monitoring, debugging, and observability + +Use structured logging, request tracing, and performance metrics to gain insights into tool execution. + +[source,yaml] +---- +include::ai-agents:example$observable_tool.yaml[] +---- + +Observability features: + +* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing +* *Execution timing*: Track how long your tools take to execute using nanosecond precision +* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` +* *Request/response metadata*: Log input parameters and response characteristics +* *Success tracking*: Monitor whether operations complete successfully + +You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: + +[source,json] +---- +{ + "metadata": { + "execution_time_ms": 0.158977, + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "success": true, + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool" + }, + "trace": { + "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", + "timestamp": "2025-09-16T08:37:18.589Z", + "tool": "observable_tool", + "version": "1.0.0" + }, + "user_id": "1" +} +---- + +=== Multi-step data enrichment + +Build tools that combine data from multiple sources. + +This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. + +[source,yaml] +---- +include::ai-agents:example$customer_enrichment.yaml[] +---- + +See also: xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) + +=== Workflow orchestration + +Coordinate complex workflows with multiple steps and conditional logic. + +This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. + +[source,yaml] +---- +include::ai-agents:example$order_workflow.yaml[] +---- + +For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: + +[source,json] +---- +{ + "assigned_rep": "vip-team@company.com", + "available_quantity": 100, + "customer_tier": "vip", + "estimated_fulfillment": "TBD - calculated based on processing tier", + "inventory_check": "passed", + "order_id": "ORD001", + "order_status": "processed", + "perks": [ + "expedited_shipping", + "white_glove_service" + ], + "priority_score": 90, + "processed_at": "2025-09-16T09:05:29.138Z", + "processing_tier": "vip", + "processing_time_estimate": "1-2 hours", + "processing_time_hours": 2, + "product_id": "widget-001", + "product_name": "Standard Widget", + "quantity": 5, + "total": 250 +} +---- + +Notice how the workflow: + +. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. +. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. +. Routes by customer tier: Since `customer_tier` is "vip", it gets VIP processing with special `perks` and priority. +. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. +. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. + +== Suggested reading + +* xref:develop:connect/components/about.adoc[Components overview] +* xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] +* xref:develop:connect/configuration/secret-management.adoc[Secret management] \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc new file mode 100644 index 000000000..dd213173d --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -0,0 +1,194 @@ += Remote MCP Server Quickstart +:page-beta: true +:description: Create and host a managed MCP server inside your Redpanda Cloud cluster with tools for generating and publishing user event data. + + +This quickstart guide shows you how to create and host a managed MCP server inside your Redpanda Cloud cluster. The server includes two tools: one for generating user event data and another for publishing that data to a Redpanda topic. + +== Prerequisites + +* A Redpanda Cloud cluster with *Remote MCP* enabled +* Access to the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] for storing credentials. +* At least version 25.2.5 of the xref:manage:rpk/rpk-install.adoc[Redpanda CLI (`rpk`)] installed on your computer +* link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code] installed + +TIP: For a complete developer guide, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. + +== Prepare your cluster + +Before creating the MCP server, you need to set up a topic and user for event publishing. Use the Redpanda CLI to perform these steps. + +. Log in to your Redpanda Cloud account: ++ +[,bash] +---- +rpk cloud login +---- ++ +This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. + +. Create a topic called `events` for storing user event data: ++ +[,bash] +---- +rpk topic create events --partitions 3 --replicas 3 +---- + +. Create a user called `mcp` with a strong password: ++ +[,bash] +---- +rpk acl user create mcp --password +---- ++ +Save the password securely. You need it later when configuring the MCP server. + +. Grant the `mcp` user permissions to produce and consume from the `events` topic: ++ +[,bash] +---- +rpk acl create --allow-principal User:mcp --operation all --topic events +---- + +== Create a Remote MCP Server in Redpanda Cloud + +. Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. + +. Navigate to *Remote MCP*. ++ +This page shows a list of existing servers. + +. Click *Create new MCP Server*. In *Metadata* add: ++ +* *Display Name*: A human-friendly name such as `event-data-generator`. This name is shown in the Redpanda Cloud Console. It is not the name of the MCP server itself. +* *Description*: Explain what the server does. For example, `Generates fake user event data and publishes it to Redpanda topics`. +* *Tags*: Add key/value tags such as `owner=platform` or `env=demo`. +* *Resources*: Choose a size (XSmall / Small / Medium / Large / XLarge). Larger sizes allow more concurrent requests and faster processing, but cost more. You can change this later. + +. Click *Next* to define tools. + +=== Add tools + +Tools define the actions your MCP server can perform. In this example, you create two tools: one for generating user event data and another for publishing that data to Redpanda. + +. From the *Template* dropdown, select *Generate Input*. ++ +The template populates the configuration with YAML for the tool definition. + +. Click *Add Tool* to create a second tool. + +. From the *Template* dropdown, select *Redpanda Output*. ++ +The template populates the configuration for publishing to Redpanda and a section for adding the required secrets is displayed. + +. Enter the values for the `mcp` user's credentials in the *Add Required Secrets* section. + +. Click *Lint* to check the configuration. You should see no errors. + +. Click *Create MCP Server* to deploy the server. ++ +It may take a few seconds to start. The status changes from *Starting* to *Running* when it's ready. + +. Open the *MCP Inspector* tab to test the tools: ++ +* For the `generate_input` tool, click *Run Tool* to generate sample event data. +* For the `redpanda_output` tool, enter some sample event data such as `user_id=user123`, `event_type=login`, and `timestamp=2025-10-21T10:30:00Z` then click *Run Tool* to publish it to the `events` topic. + +== Connect a client + +You can connect any MCP-compatible client to your Remote MCP server. This example uses Claude Code. + +The Redpanda CLI acts as a local proxy, forwarding requests from your AI client to your own Remote MCP server running in the Redpanda Cloud cluster. + +. Log in to your Redpanda Cloud account: ++ +[,bash] +---- +rpk cloud login +---- ++ +This opens a browser window to authenticate. The token is saved locally inside your `rpk` configuration file. It is valid for 4 hours. You can refresh it by running `rpk cloud login` again. + +. Open the *Connection* tab in Redpanda Cloud to get connection details and run the `rpk` command for Claude Code. ++ +For BYOC and Dedicated clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--cluster-id \ +--mcp-server-id \ +--install --client claude-code +---- ++ +For Serverless clusters, use: ++ +[,bash] +---- +rpk cloud mcp proxy \ +--serverless-cluster-id \ +--mcp-server-id \ +--install --client claude-code +---- + +. Restart Claude Code and invoke your tool. ++ +[,bash] +---- +claude +---- + +. Ask Claude Code to use your tools. For example: ++ +* "Generate 10 user events and then publish them to the events topic." +* "Create sample login events for users user001, user002, and user003, then publish them to Redpanda." +* "Generate purchase events with metadata and publish them to the events topic." + +You should see Claude Code calling your MCP server tools to generate and publish event data. You may need to respond to prompts for permissions to call the tools. + +When complete, you can verify the events were published by consuming from the `events` topic: + +[,bash] +---- +rpk topic consume events --num 10 +---- + +== Troubleshoot + +=== MCP server not starting + +If your server is not starting or shows an error state: + +- Check the *Logs* tab for specific error messages. +- Verify your YAML syntax by clicking *Lint*. +- Verify that the `mcp` user exists and has the correct permissions. + +=== Connection issues + +If Claude Code can't connect to your server: + +- Verify you're logged in with `rpk cloud login`. +- Check that your rpk version is 25.2.5 or later: `rpk version`. +- Ensure your server status shows *Running* in Redpanda Cloud. +- Try restarting Claude Code after running the proxy command. + +=== Publishing failures + +If the publish tool returns errors: + +- Verify the `events` topic exists: ++ +```bash +rpk topic list +``` +- Check that the `mcp` user has the correct ACLs: ++ +```bash +rpk acl list +``` +- Use the MCP Inspector in Redpanda Cloud to test with different event data. +- Check the MCP server logs for authentication or authorization errors. + +== Next steps + +Write your own tools to extend the MCP server functionality. For more information, see: xref:ai-agents:mcp/remote/developer-guide.adoc[] diff --git a/modules/get-started/pages/cloud-overview.adoc b/modules/get-started/pages/cloud-overview.adoc index 7f33c5c16..7691345ae 100644 --- a/modules/get-started/pages/cloud-overview.adoc +++ b/modules/get-started/pages/cloud-overview.adoc @@ -387,7 +387,7 @@ The following features are currently in beta in Redpanda Cloud: * BYOVNet for Azure * Secrets Management for BYOVPC on GCP and AWS * Remote Read Replicas for AWS and GCP -* Redpanda Cloud MCP Server +* xref:ai-agents:index.adoc[AI agents with MCP servers] * Several Redpanda Connect components == Next steps diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index 60443e5cb..599847e52 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -8,6 +8,14 @@ This page lists new features added to Redpanda Cloud. == October 2025 +=== Remote MCP: beta + +Deploy managed MCP servers directly inside your Redpanda Cloud cluster with xref:ai-agents:mcp/remote/overview.adoc[Remote MCP]. Unlike the Redpanda Cloud MCP Server, Remote MCP servers run within your cluster and can process data streams, generate synthetic data, and publish directly to Redpanda topics. + +Create custom AI tools using templates or write your own Redpanda Connect configurations to build event-driven workflows. Remote MCP servers provide AI assistants with streaming data capabilities, enabling use cases like real-time data generation, stream processing, and event publishing. + +Get started with the xref:ai-agents:mcp/remote/quickstart.adoc[quickstart guide] or explore advanced patterns in the xref:ai-agents:mcp/remote/developer-guide.adoc[developer guide]. + === API Gateway access BYOC and Dedicated clusters with private networking now allow control of API Gateway network access, independent of the Redpanda cluster. When you create a cluster, you can choose either public or private access for the API Gateway: diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc index d94f56b26..286dc30dc 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc @@ -1,3 +1,4 @@ = rpk cloud mcp install +:description: Install the local MCP server for Redpanda Cloud configuration. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-install.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc new file mode 100644 index 000000000..b38ce0fe2 --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc @@ -0,0 +1,4 @@ += rpk cloud mcp proxy +:description: Proxy MCP requests to remote MCP servers in Redpanda Cloud. + +include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-proxy.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc index b440addd6..aa1f11f42 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc @@ -1,3 +1,4 @@ = rpk cloud mcp stdio +:description: Communicate with the local MCP server for Redpanda Cloud using the stdio protocol. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-stdio.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc new file mode 100644 index 000000000..0cbf45193 --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp.adoc @@ -0,0 +1,4 @@ += rpk cloud mcp +:description: Manage connections to MCP servers in Redpanda Cloud. + +include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp.adoc[tag=single-source] \ No newline at end of file