From f19bff6de441baa9d5096c920695ee4131b75928 Mon Sep 17 00:00:00 2001 From: Johnathon Selstad Date: Tue, 24 Feb 2026 12:27:15 -0800 Subject: [PATCH 1/4] keynote-2: add fair benchmark variant with leveled playing field Add an alternative benchmark configuration that eliminates compounding asymmetries between SpacetimeDB and competitors: - TypeScript client for ALL systems (no custom Rust client for SpacetimeDB) - confirmedReads=true for SpacetimeDB (durable commits, matching Postgres fsync) - Client-side TPS counting for all (no server-side Prometheus metrics) - Same pipeline depth (8) for all systems - Postgres read_committed isolation (its actual default, not serializable) - Postgres synchronous_commit=on (matching SpacetimeDB durability) - New stored-procedure RPC server eliminates Drizzle ORM multi-round-trip overhead, making Postgres comparable to SpacetimeDB's single-call reducer New files: - src/fair-bench.ts: fair benchmark runner with enforced equal settings - src/rpc-servers/postgres-storedproc-rpc-server.ts: PL/pgSQL stored proc - src/connectors/rpc/postgres_storedproc_rpc.ts: connector for above - docker-compose-fair.yml: Postgres with fair configuration - FAIR-BENCHMARK.md: methodology documentation Local test results (Postgres only, same machine): - Postgres (Drizzle ORM): 1,817 TPS @ alpha=0.5, 815 TPS @ alpha=1.5 - Postgres (stored proc): 3,415 TPS @ alpha=0.5, 1,217 TPS @ alpha=1.5 The stored procedure alone provides ~1.9x speedup by eliminating ORM round-trips -- demonstrating that a significant portion of the original benchmark's gap comes from penalizing competitors with unnecessary overhead. Co-Authored-By: Claude Opus 4.6 (1M context) --- templates/keynote-2/FAIR-BENCHMARK.md | 114 +++++ templates/keynote-2/docker-compose-fair.yml | 82 ++++ templates/keynote-2/package.json | 3 +- templates/keynote-2/src/connectors/index.ts | 2 + .../connectors/rpc/postgres_storedproc_rpc.ts | 98 +++++ templates/keynote-2/src/fair-bench.ts | 402 ++++++++++++++++++ .../postgres-storedproc-rpc-server.ts | 296 +++++++++++++ .../tests/test-1/postgres_storedproc_rpc.ts | 7 + 8 files changed, 1003 insertions(+), 1 deletion(-) create mode 100644 templates/keynote-2/FAIR-BENCHMARK.md create mode 100644 templates/keynote-2/docker-compose-fair.yml create mode 100644 templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts create mode 100644 templates/keynote-2/src/fair-bench.ts create mode 100644 templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts create mode 100644 templates/keynote-2/src/tests/test-1/postgres_storedproc_rpc.ts diff --git a/templates/keynote-2/FAIR-BENCHMARK.md b/templates/keynote-2/FAIR-BENCHMARK.md new file mode 100644 index 00000000000..1b3db5f7ff6 --- /dev/null +++ b/templates/keynote-2/FAIR-BENCHMARK.md @@ -0,0 +1,114 @@ +# Fair Benchmark: SpacetimeDB vs Competitors + +This is an alternative benchmark configuration that levels the playing field between SpacetimeDB and traditional database stacks. The original benchmark (`demo.ts`) has several asymmetries that compound to inflate SpacetimeDB's advantage far beyond what the architecture alone provides. + +## What This Changes + +| Factor | Original Benchmark | Fair Benchmark | +|--------|-------------------|----------------| +| **SpacetimeDB client** | Custom Rust client with 16,384 in-flight ops | Same TypeScript client as everyone else | +| **TPS counting** | Server-side Prometheus metrics (fire-and-forget) | Client-side round-trip counting for ALL systems | +| **Durability** | `confirmedReads=false` (no durability guarantee) | `confirmedReads=true` (durable commits, like Postgres fsync) | +| **Pipeline depth** | 16,384 for SpacetimeDB vs 8 for competitors | Same for all (configurable, default 8) | +| **Postgres isolation** | `serializable` (non-default, worst-case for contention) | `read_committed` (Postgres actual default) | +| **Postgres sync commit** | `synchronous_commit=off` | `synchronous_commit=on` (matches SpacetimeDB confirmed reads) | +| **Postgres transfer** | 4 ORM round-trips via Drizzle | Also tested with stored procedure (single DB call) | +| **Warmup** | 5s warmup for Rust client only | No warmup for any system (equal cold start) | + +## Why These Changes Matter + +### 1. Same Client Language (TypeScript for All) + +The original benchmark uses a hand-tuned **Rust client** for SpacetimeDB that sends 16,384 concurrent operations per connection via binary WebSocket, while all competitors use a TypeScript client with HTTP/JSON and 8 in-flight operations. This alone is a ~2000x difference in pipeline depth. + +The README justifies this by saying "we were bottlenecked on our test TypeScript client" — but then no competitor gets the same optimization. A fair comparison uses the same client for all. + +### 2. Confirmed Reads (Durable Commits) + +The original benchmark defaults `STDB_CONFIRMED_READS` to `false`, meaning SpacetimeDB doesn't wait for durable commits before reporting success. Meanwhile Postgres runs with `fsync=on`. This is comparing "maybe durable" vs "definitely durable" — not a fair durability comparison. + +### 3. Client-Side TPS Counting + +The original `demo.ts` sets `USE_SPACETIME_METRICS_ENDPOINT=1`, which counts committed transactions **on the server** via Prometheus. Combined with the fire-and-forget Rust client, this counts transactions that completed server-side but whose acknowledgments may not have reached the client. All other systems count only after the full round-trip completes. + +### 4. Postgres Isolation Level + +The original forces `default_transaction_isolation=serializable` on Postgres, which is **not** the default (`read_committed`). Under the Zipf contention workload, serializable causes massive transaction aborts and retries, dramatically hurting Postgres performance. The benchmark already uses `SELECT ... FOR UPDATE` for row-level locking, making serializable unnecessary. + +### 5. Stored Procedure vs ORM + +SpacetimeDB's reducer executes as a single atomic operation inside the database. The original Postgres benchmark uses Drizzle ORM which requires: +- `BEGIN` +- `SELECT ... FOR UPDATE` (fetch both accounts) +- `UPDATE` (debit) +- `UPDATE` (credit) +- `COMMIT` + +That's 5 round-trips between the Node.js process and Postgres. A stored procedure (`do_transfer()`) does the same work in a single call — which is the fair equivalent of SpacetimeDB's reducer model. + +## Running the Fair Benchmark + +### Prerequisites + +```bash +# Install dependencies +pnpm install + +# Start services +docker compose -f docker-compose-fair.yml up -d + +# Start SpacetimeDB +spacetime start + +# Publish the SpacetimeDB module +spacetime publish --server local test-1 --module-path ./spacetimedb +``` + +### Run + +```bash +# Default: SpacetimeDB vs Postgres (ORM) vs Postgres (stored proc) +npm run fair-bench + +# With options +npm run fair-bench -- --seconds 10 --concurrency 50 --alpha 0.5 + +# High contention +npm run fair-bench -- --alpha 1.5 + +# Include more systems +npm run fair-bench -- --systems spacetimedb,postgres_rpc,postgres_storedproc_rpc,sqlite_rpc + +# Custom pipeline depth +npm run fair-bench -- --pipeline-depth 16 + +# Skip seeding (if already seeded) +npm run fair-bench -- --skip-prep +``` + +### Start the Stored Procedure RPC Server (non-Docker) + +```bash +# Set PG_URL in .env or environment +PG_STOREDPROC_RPC_PORT=4105 npx tsx src/rpc-servers/postgres-storedproc-rpc-server.ts +``` + +## Expected Results + +With a leveled playing field, SpacetimeDB's genuine architectural advantage (colocated compute+storage, no network hop for data access) should still show a meaningful speedup — likely **2-5x** rather than the claimed **14x**. The remaining advantage is real and architectural: + +- Zero-copy in-process data access vs TCP round-trips +- Rust execution vs Node.js JavaScript +- Binary BSATN protocol vs JSON serialization + +The factors that are **not** architectural and were removed: +- Custom Rust client vs shared TypeScript client +- 16,384 vs 8 pipeline depth +- Server-side vs client-side TPS counting +- Unequal durability guarantees +- Non-default Postgres isolation level penalizing competitors +- ORM overhead (5 round-trips) vs single reducer call + +## Detailed Asymmetry Analysis + +For a comprehensive breakdown of every asymmetry in the original benchmark, see the table in the PR description. diff --git a/templates/keynote-2/docker-compose-fair.yml b/templates/keynote-2/docker-compose-fair.yml new file mode 100644 index 00000000000..48ade3a59af --- /dev/null +++ b/templates/keynote-2/docker-compose-fair.yml @@ -0,0 +1,82 @@ +# Fair benchmark configuration +# +# Key differences from docker-compose.yml: +# 1. Postgres uses read_committed isolation (its actual default) instead of serializable +# 2. synchronous_commit=on for Postgres (matches SpacetimeDB confirmed reads) +# 3. Stored procedure RPC server (eliminates ORM round-trip overhead) +# +# Run with: docker compose -f docker-compose-fair.yml up -d + +services: + pg-fair: + image: postgres:16 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "5432:5432" + command: > + -c fsync=on + -c synchronous_commit=on + -c shared_buffers=8GB + -c work_mem=64MB + -c max_connections=10000 + -c default_transaction_isolation=read\ committed + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 15 + volumes: + - pg_fair_data:/var/lib/postgresql/data + network_mode: host + + # Original Drizzle ORM RPC server (for comparison) + pg-rpc-fair: + build: + context: . + dockerfile: Dockerfile.rpc + command: ["pnpm", "tsx", "src/rpc-servers/postgres-rpc-server.ts"] + ports: + - "4101:4101" + environment: + PG_URL: ${PG_URL} + PG_RPC_PORT: "4101" + SEED_ACCOUNTS: ${SEED_ACCOUNTS} + SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} + depends_on: + pg-fair: + condition: service_healthy + network_mode: host + + # Stored procedure RPC server (fair comparison) + pg-storedproc-rpc-fair: + build: + context: . + dockerfile: Dockerfile.rpc + command: ["pnpm", "tsx", "src/rpc-servers/postgres-storedproc-rpc-server.ts"] + ports: + - "4105:4105" + environment: + PG_URL: ${PG_URL} + PG_STOREDPROC_RPC_PORT: "4105" + SEED_ACCOUNTS: ${SEED_ACCOUNTS} + SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} + depends_on: + pg-fair: + condition: service_healthy + network_mode: host + + spacetime-fair: + image: clockworklabs/spacetime:latest + command: start + ports: + - "3000:3000" + volumes: + - spacetime_fair_data:/data + network_mode: host + +volumes: + pg_fair_data: + spacetime_fair_data: diff --git a/templates/keynote-2/package.json b/templates/keynote-2/package.json index c19cf90fd43..e8af28bd4fe 100644 --- a/templates/keynote-2/package.json +++ b/templates/keynote-2/package.json @@ -11,7 +11,8 @@ "prep": "tsx src/init/init-all.ts", "down": "docker compose down || exit 0", "test-1": "tsx src/cli.ts test-1", - "bench": "tsx src/cli.ts" + "bench": "tsx src/cli.ts", + "fair-bench": "tsx src/fair-bench.ts" }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", diff --git a/templates/keynote-2/src/connectors/index.ts b/templates/keynote-2/src/connectors/index.ts index 14a57e4d360..3a618cd4eb1 100644 --- a/templates/keynote-2/src/connectors/index.ts +++ b/templates/keynote-2/src/connectors/index.ts @@ -6,6 +6,7 @@ import cockroach_rpc from './rpc/cockroach_rpc.ts'; import sqlite_rpc from './rpc/sqlite_rpc.ts'; import supabase_rpc from './rpc/supabase_rpc.ts'; import planetscale_pg_rpc from './rpc/planetscale_pg_rpc.ts'; +import postgres_storedproc_rpc from './rpc/postgres_storedproc_rpc.ts'; export const CONNECTORS = { convex, @@ -16,5 +17,6 @@ export const CONNECTORS = { sqlite_rpc, supabase_rpc, planetscale_pg_rpc, + postgres_storedproc_rpc, }; export type ConnectorKey = keyof typeof CONNECTORS; diff --git a/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts b/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts new file mode 100644 index 00000000000..b4be0bfe102 --- /dev/null +++ b/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts @@ -0,0 +1,98 @@ +import type { RpcConnector } from '../../core/connectors.ts'; +import { RpcRequest, RpcResponse } from './rpc_common.ts'; + +/** + * Connector for the stored-procedure Postgres RPC server. + * Identical interface to postgres_rpc, just points at a different port. + */ +export default function postgres_storedproc_rpc( + url = process.env.PG_STOREDPROC_RPC_URL || 'http://127.0.0.1:4105', +): RpcConnector { + let rpcUrl: URL | null = null; + + function ensureUrl() { + if (!url) throw new Error('PG_STOREDPROC_RPC_URL not set'); + if (!rpcUrl) rpcUrl = new URL('/rpc', new URL(url)); + } + + async function httpCall(name: string, args?: Record) { + ensureUrl(); + const body: RpcRequest = { name, args }; + + const res = await fetch(rpcUrl!, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + + const text = await res.text(); + let json: RpcResponse; + try { + json = JSON.parse(text) as RpcResponse; + } catch { + throw new Error( + `[postgres_storedproc_rpc] invalid JSON: HTTP ${res.status} ${res.statusText} body=${text.slice( + 0, + 200, + )}`, + ); + } + + if (!res.ok || !json.ok) { + throw new Error( + `[postgres_storedproc_rpc] RPC ${name} failed: HTTP ${res.status} ${res.statusText} body=${text.slice( + 0, + 200, + )}`, + ); + } + + return json.result; + } + + const root: RpcConnector = { + name: 'postgres_storedproc_rpc', + + async open() { + await httpCall('health'); + }, + + async close() { + // no-op + }, + + async getAccount(id: number) { + const result = (await httpCall('getAccount', { id })) as { + id: number; + balance: bigint; + } | null; + + if (!result) return null; + return { + id: result.id, + balance: result.balance, + }; + }, + + async verify() { + await httpCall('verify'); + }, + + async call(name: string, args?: Record) { + return httpCall(name, args); + }, + + async createWorker(): Promise { + const worker = postgres_storedproc_rpc(url); + worker.verify = async () => { + throw new Error( + 'verify() not supported on postgres_storedproc_rpc worker; call verify() on root', + ); + }; + delete worker.createWorker; + return worker as RpcConnector; + }, + }; + + return root; +} diff --git a/templates/keynote-2/src/fair-bench.ts b/templates/keynote-2/src/fair-bench.ts new file mode 100644 index 00000000000..71af289d8b6 --- /dev/null +++ b/templates/keynote-2/src/fair-bench.ts @@ -0,0 +1,402 @@ +/** + * Fair Benchmark Runner + * + * Levels the playing field between SpacetimeDB and competitors by: + * + * 1. Using the TypeScript client for ALL systems (no custom Rust client) + * 2. Forcing STDB_CONFIRMED_READS=1 (durable commits, like Postgres with fsync) + * 3. Forcing USE_SPACETIME_METRICS_ENDPOINT=0 (client-side TPS counting for all) + * 4. Using the same pipeline depth for all systems + * 5. Including postgres_storedproc_rpc (stored procedure, eliminates ORM overhead) + * 6. Using read_committed isolation for Postgres (its actual default) + * + * Usage: + * npx tsx src/fair-bench.ts [--seconds N] [--concurrency N] [--alpha N] [--systems a,b,c] + * + * Default systems: spacetimedb,postgres_rpc,postgres_storedproc_rpc + */ + +import 'dotenv/config'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { createConnection } from 'node:net'; +import { join } from 'node:path'; +import { CONNECTORS } from './connectors'; +import { runOne } from './core/runner'; + +// ============================================================================ +// Force fair settings +// ============================================================================ + +// Force SpacetimeDB to use confirmed reads (durable commits) +process.env.STDB_CONFIRMED_READS = '1'; + +// Force client-side TPS counting (no server-side metrics cheating) +process.env.USE_SPACETIME_METRICS_ENDPOINT = '0'; + +// Non-docker mode +process.env.USE_DOCKER = '0'; + +// Set default SpacetimeDB config if not set +if (!process.env.STDB_URL) process.env.STDB_URL = 'ws://127.0.0.1:3000'; +if (!process.env.STDB_MODULE) process.env.STDB_MODULE = 'test-1'; + +// ============================================================================ +// CLI Arguments +// ============================================================================ + +const args = process.argv.slice(2); + +function getArg(name: string, defaultValue: number): number { + const idx = args.findIndex( + (a) => a === `--${name}` || a.startsWith(`--${name}=`), + ); + if (idx === -1) return defaultValue; + const arg = args[idx]; + if (arg.includes('=')) return Number(arg.split('=')[1]); + return Number(args[idx + 1] ?? defaultValue); +} + +function getStringArg(name: string, defaultValue: string): string { + const idx = args.findIndex( + (a) => a === `--${name}` || a.startsWith(`--${name}=`), + ); + if (idx === -1) return defaultValue; + const arg = args[idx]; + if (arg.includes('=')) return arg.split('=')[1]; + return args[idx + 1] ?? defaultValue; +} + +function hasFlag(name: string): boolean { + return args.includes(`--${name}`); +} + +const seconds = getArg('seconds', 10); +const concurrency = getArg('concurrency', 50); +const alpha = getArg('alpha', 0.5); +const systems = getStringArg( + 'systems', + 'spacetimedb,postgres_rpc,postgres_storedproc_rpc', +) + .split(',') + .map((s) => s.trim()); +const pipelineDepth = getArg('pipeline-depth', 8); +const skipPrep = hasFlag('skip-prep'); + +const accounts = Number(process.env.SEED_ACCOUNTS ?? 100_000); +const initialBalance = Number(process.env.SEED_INITIAL_BALANCE ?? 10_000_000); + +// Set the same pipeline depth for all systems +process.env.MAX_INFLIGHT_PER_WORKER = String(pipelineDepth); + +// ============================================================================ +// ANSI Colors +// ============================================================================ + +const colors = { + reset: '\x1b[0m', + bold: '\x1b[1m', + dim: '\x1b[2m', + green: '\x1b[32m', + yellow: '\x1b[33m', + cyan: '\x1b[36m', + red: '\x1b[31m', +}; + +function c(color: keyof typeof colors, text: string): string { + return `${colors[color]}${text}${colors.reset}`; +} + +// ============================================================================ +// Health Checks +// ============================================================================ + +function ping(port: number, timeoutMs = 2000): Promise { + return new Promise((resolve) => { + const socket = createConnection({ host: '127.0.0.1', port }); + const timer = setTimeout(() => { + socket.destroy(); + resolve(false); + }, timeoutMs); + socket.on('connect', () => { + clearTimeout(timer); + socket.destroy(); + resolve(true); + }); + socket.on('error', () => { + clearTimeout(timer); + resolve(false); + }); + }); +} + +const serviceChecks: Record = { + spacetimedb: { + name: 'SpacetimeDB', + port: 3000, + hint: 'spacetime start', + }, + postgres_rpc: { + name: 'Postgres (Drizzle ORM)', + port: 4101, + hint: 'docker compose -f docker-compose-fair.yml up -d pg-fair pg-rpc-fair', + }, + postgres_storedproc_rpc: { + name: 'Postgres (Stored Proc)', + port: 4105, + hint: 'docker compose -f docker-compose-fair.yml up -d pg-fair pg-storedproc-rpc-fair', + }, + sqlite_rpc: { + name: 'SQLite RPC', + port: 4103, + hint: 'npx tsx src/rpc-servers/sqlite-rpc-server.ts', + }, + cockroach_rpc: { + name: 'CockroachDB RPC', + port: 4102, + hint: 'docker compose up -d crdb crdb-rpc', + }, + convex: { + name: 'Convex', + port: 3210, + hint: 'cd convex-app && npx convex dev', + }, +}; + +// ============================================================================ +// Prep / Seed +// ============================================================================ + +async function prepSystem(system: string): Promise { + const connectorFactory = (CONNECTORS as any)[system]; + if (!connectorFactory) { + console.log(` ${system.padEnd(28)} ${c('yellow', 'SKIPPED (unknown)')}`); + return; + } + + try { + if (system === 'spacetimedb') { + const conn = connectorFactory(); + await conn.open(); + await conn.reducer('seed', { + n: accounts, + initial_balance: BigInt(initialBalance), + }); + await conn.close(); + } else { + const conn = connectorFactory(); + await conn.open(); + await conn.call('seed', { accounts, initialBalance }); + await conn.close(); + } + console.log(` ${system.padEnd(28)} ${c('green', 'SEEDED')}`); + } catch (err: any) { + console.log(` ${system.padEnd(28)} ${c('red', `FAILED: ${err.message}`)}`); + } +} + +// ============================================================================ +// Benchmark +// ============================================================================ + +interface BenchResult { + system: string; + tps: number; + p50_ms: number; + p95_ms: number; + p99_ms: number; + samples: number; +} + +async function runBenchmark(system: string): Promise { + const connectorFactory = (CONNECTORS as any)[system]; + if (!connectorFactory) { + console.log(` ${system}: Unknown connector`); + return null; + } + + const connector = connectorFactory(); + + // Load the test scenario + let scenario: any; + try { + const testMod = await import(`./tests/test-1/${system}.ts`); + scenario = testMod.default.run; + } catch { + // Fallback to rpc_single_call for RPC-based systems + const { rpc_single_call } = await import('./scenario_recipes/rpc_single_call.ts'); + scenario = rpc_single_call; + } + + const result = await runOne({ + connector, + scenario, + seconds, + concurrency, + accounts, + alpha, + }); + + return { + system, + tps: Math.round(result.tps), + p50_ms: result.p50_ms, + p95_ms: result.p95_ms, + p99_ms: result.p99_ms, + samples: result.samples, + }; +} + +// ============================================================================ +// Display +// ============================================================================ + +function renderBar(tps: number, maxTps: number, width = 40): string { + const filled = Math.max(1, Math.round((tps / maxTps) * width)); + return c('green', '\u2588'.repeat(filled) + '\u2591'.repeat(width - filled)); +} + +// ============================================================================ +// Main +// ============================================================================ + +async function main() { + console.log(''); + console.log(c('bold', c('cyan', ' Fair Benchmark: SpacetimeDB vs Competitors'))); + console.log(c('dim', ' Leveled playing field - same client, same durability, same counting')); + console.log(''); + + console.log(c('bold', ' Configuration:')); + console.log(` Duration: ${seconds}s`); + console.log(` Concurrency: ${concurrency} connections`); + console.log(` Alpha (contention): ${alpha}`); + console.log(` Pipeline depth: ${pipelineDepth} per worker (same for all)`); + console.log(` Systems: ${systems.join(', ')}`); + console.log(''); + + console.log(c('bold', ' Fairness guarantees:')); + console.log(` ${c('green', '\u2713')} TypeScript client for ALL systems (no custom Rust client)`); + console.log(` ${c('green', '\u2713')} STDB_CONFIRMED_READS=1 (durable commits)`); + console.log(` ${c('green', '\u2713')} Client-side TPS counting for ALL systems`); + console.log(` ${c('green', '\u2713')} Same pipeline depth (${pipelineDepth}) for all`); + console.log(` ${c('green', '\u2713')} Postgres: read_committed isolation (actual default)`); + console.log(` ${c('green', '\u2713')} Postgres: synchronous_commit=on`); + console.log(''); + + // Check services + console.log(c('bold', ' [1/3] Checking services...\n')); + for (const system of systems) { + const check = serviceChecks[system]; + if (!check) { + console.log(` ${system.padEnd(28)} ${c('yellow', '? (no health check)')}`); + continue; + } + const alive = await ping(check.port); + if (alive) { + console.log(` ${check.name.padEnd(28)} ${c('green', 'UP')}`); + } else { + console.log(` ${check.name.padEnd(28)} ${c('red', 'DOWN')}`); + console.log(` Start with: ${c('cyan', check.hint)}`); + process.exit(1); + } + } + + // Seed + if (!skipPrep) { + console.log('\n' + c('bold', ' [2/3] Seeding databases...\n')); + for (const system of systems) { + await prepSystem(system); + } + } else { + console.log('\n' + c('bold', ' [2/3] Seeding... ') + c('dim', '(skipped)\n')); + } + + // Benchmark + console.log('\n' + c('bold', ' [3/3] Running benchmarks...\n')); + + const results: BenchResult[] = []; + for (const system of systems) { + console.log(` Running ${system}...`); + const result = await runBenchmark(system); + if (result && result.tps > 0) { + console.log(` ${system.padEnd(28)} ${c('green', `${result.tps.toLocaleString()} TPS`)} (p50=${result.p50_ms.toFixed(1)}ms p95=${result.p95_ms.toFixed(1)}ms p99=${result.p99_ms.toFixed(1)}ms)`); + results.push(result); + } else { + console.log(` ${system.padEnd(28)} ${c('red', 'FAILED')}`); + } + } + + // Results + if (results.length > 0) { + results.sort((a, b) => b.tps - a.tps); + const maxTps = results[0]?.tps || 1; + + console.log('\n' + c('bold', '\u2550'.repeat(70))); + console.log(c('bold', ' FAIR BENCHMARK RESULTS')); + console.log(c('bold', '\u2550'.repeat(70)) + '\n'); + + for (const r of results) { + const bar = renderBar(r.tps, maxTps); + const tpsStr = r.tps.toLocaleString().padStart(10); + console.log(` ${r.system.padEnd(28)} ${bar} ${tpsStr} TPS`); + } + + const fastest = results[0]; + const slowest = results[results.length - 1]; + + if (fastest && slowest && fastest.system !== slowest.system && slowest.tps > 0) { + const multiplier = (fastest.tps / slowest.tps).toFixed(1); + console.log(''); + console.log(` ${fastest.system} is ${multiplier}x faster than ${slowest.system}`); + } + + // Latency table + console.log('\n' + c('bold', ' Latency (ms):')); + console.log(` ${'System'.padEnd(28)} ${'p50'.padStart(8)} ${'p95'.padStart(8)} ${'p99'.padStart(8)}`); + console.log(` ${'-'.repeat(28)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)}`); + for (const r of results) { + console.log(` ${r.system.padEnd(28)} ${r.p50_ms.toFixed(1).padStart(8)} ${r.p95_ms.toFixed(1).padStart(8)} ${r.p99_ms.toFixed(1).padStart(8)}`); + } + + // Save to JSON + const runsDir = join(process.cwd(), 'runs'); + await mkdir(runsDir, { recursive: true }); + const outFile = join( + runsDir, + `fair-bench-${new Date().toISOString().replace(/[:.]/g, '-')}.json`, + ); + await writeFile( + outFile, + JSON.stringify( + { + benchmark: 'fair-comparison', + timestamp: new Date().toISOString(), + fairness: { + confirmed_reads: true, + metrics_endpoint: false, + client: 'typescript (same for all)', + pipeline_depth: pipelineDepth, + postgres_isolation: 'read_committed', + postgres_synchronous_commit: 'on', + }, + config: { seconds, concurrency, alpha, accounts, pipelineDepth }, + results: results.map((r) => ({ + system: r.system, + tps: r.tps, + p50_ms: r.p50_ms, + p95_ms: r.p95_ms, + p99_ms: r.p99_ms, + samples: r.samples, + })), + }, + null, + 2, + ), + ); + console.log(`\n${c('dim', ` Results saved to: ${outFile}`)}\n`); + } +} + +main().catch((err) => { + console.error('\n' + c('red', ' ERROR:'), err.message); + process.exit(1); +}); diff --git a/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts b/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts new file mode 100644 index 00000000000..37b4a2c52c9 --- /dev/null +++ b/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts @@ -0,0 +1,296 @@ +import 'dotenv/config'; +import http from 'node:http'; +import { Pool } from 'pg'; +import { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts'; +import { poolMaxFromEnv } from '../helpers.ts'; + +/** + * Fair benchmark variant: Postgres RPC server using a stored procedure + * for the transfer operation instead of Drizzle ORM with multiple round-trips. + * + * This eliminates the ORM overhead and reduces the transfer from 4 SQL + * round-trips (BEGIN + SELECT FOR UPDATE + UPDATE + UPDATE + COMMIT via Drizzle) + * to a single SQL call (SELECT do_transfer(...)). + * + * This is analogous to what SpacetimeDB does: a single reducer call that + * executes atomically inside the database engine. + */ + +const PG_URL = process.env.PG_URL; +if (!PG_URL) { + throw new Error('PG_URL not set'); +} + +const pool = new Pool({ + connectionString: PG_URL, + application_name: 'pg-storedproc-rpc', + max: poolMaxFromEnv(), +}); + +async function ensureStoredProcedure() { + const client = await pool.connect(); + try { + // Create the stored procedure if it doesn't exist. + // This does the exact same logic as the Drizzle version but in a single + // database call with zero ORM overhead. + await client.query(` + CREATE OR REPLACE FUNCTION do_transfer( + p_from_id INTEGER, + p_to_id INTEGER, + p_amount BIGINT + ) RETURNS VOID AS $$ + DECLARE + v_from_balance BIGINT; + v_to_balance BIGINT; + BEGIN + IF p_from_id = p_to_id OR p_amount <= 0 THEN + RETURN; + END IF; + + -- Row-level lock with consistent ordering to avoid deadlocks + IF p_from_id < p_to_id THEN + SELECT balance INTO v_from_balance FROM accounts WHERE id = p_from_id FOR UPDATE; + SELECT balance INTO v_to_balance FROM accounts WHERE id = p_to_id FOR UPDATE; + ELSE + SELECT balance INTO v_to_balance FROM accounts WHERE id = p_to_id FOR UPDATE; + SELECT balance INTO v_from_balance FROM accounts WHERE id = p_from_id FOR UPDATE; + END IF; + + IF v_from_balance IS NULL OR v_to_balance IS NULL THEN + RAISE EXCEPTION 'account_missing'; + END IF; + + IF v_from_balance < p_amount THEN + RETURN; -- insufficient funds, silently skip (matches original behavior) + END IF; + + UPDATE accounts SET balance = balance - p_amount WHERE id = p_from_id; + UPDATE accounts SET balance = balance + p_amount WHERE id = p_to_id; + END; + $$ LANGUAGE plpgsql; + `); + console.log('[pg-storedproc-rpc] stored procedure do_transfer() created/updated'); + } finally { + client.release(); + } +} + +async function rpcTransfer(args: Record) { + const fromId = Number(args.from_id ?? args.from); + const toId = Number(args.to_id ?? args.to); + const amount = Number(args.amount); + + if ( + !Number.isInteger(fromId) || + !Number.isInteger(toId) || + !Number.isFinite(amount) + ) { + throw new Error('invalid transfer args'); + } + if (fromId === toId || amount <= 0) return; + + // Single database call - no ORM, no multiple round-trips + await pool.query('SELECT do_transfer($1, $2, $3)', [fromId, toId, BigInt(amount)]); +} + +async function rpcGetAccount(args: Record) { + const id = Number(args.id); + if (!Number.isInteger(id)) throw new Error('invalid id'); + + const result = await pool.query( + 'SELECT id, balance FROM accounts WHERE id = $1 LIMIT 1', + [id], + ); + + if (result.rows.length === 0) return null; + const row = result.rows[0]!; + return { + id: row.id, + balance: row.balance.toString(), + }; +} + +async function rpcVerify() { + const rawInitial = process.env.SEED_INITIAL_BALANCE; + if (!rawInitial) { + console.warn('[pg-storedproc-rpc] SEED_INITIAL_BALANCE not set; skipping verify'); + return { skipped: true }; + } + + let initial: bigint; + try { + initial = BigInt(rawInitial); + } catch { + throw new Error(`invalid SEED_INITIAL_BALANCE=${rawInitial}`); + } + + const result = await pool.query(` + SELECT + COUNT(*)::bigint AS count, + COALESCE(SUM("balance"), 0)::bigint AS total, + SUM( + CASE WHEN "balance" != $1::bigint THEN 1 ELSE 0 END + )::bigint AS changed + FROM accounts + `, [initial]); + + const row = result.rows[0] as { + count: string | number | bigint; + total: string | number | bigint; + changed: string | number | bigint; + }; + + const count = BigInt(row.count); + const total = BigInt(row.total); + const changed = BigInt(row.changed); + const expected = initial * count; + + if (count === 0n) throw new Error('verify failed: accounts=0'); + if (total !== expected) { + throw new Error( + `verify failed: accounts=${count} total=${total} expected=${expected}`, + ); + } + if (changed === 0n) { + throw new Error('verify failed: total preserved but no balances changed'); + } + + return { + accounts: count.toString(), + total: total.toString(), + changed: changed.toString(), + }; +} + +async function rpcSeed(args: Record) { + const count = Number(args.accounts ?? process.env.SEED_ACCOUNTS ?? '0'); + const rawInitial = + (args.initialBalance as string | number | undefined) ?? + process.env.SEED_INITIAL_BALANCE; + + if (!Number.isInteger(count) || count <= 0) { + throw new Error('[pg-storedproc-rpc] invalid accounts for seed'); + } + if (rawInitial === undefined || rawInitial === null) { + throw new Error('[pg-storedproc-rpc] missing initialBalance for seed'); + } + + let initial: bigint; + try { + initial = BigInt(rawInitial); + } catch { + throw new Error(`[pg-storedproc-rpc] invalid initialBalance=${rawInitial}`); + } + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query('DELETE FROM accounts'); + + const batchSize = 10_000; + for (let start = 0; start < count; start += batchSize) { + const end = Math.min(start + batchSize, count); + const values: string[] = []; + const params: any[] = []; + let paramIdx = 1; + + for (let id = start; id < end; id++) { + values.push(`($${paramIdx++}, $${paramIdx++})`); + params.push(id, initial); + } + + await client.query( + `INSERT INTO accounts (id, balance) VALUES ${values.join(', ')}`, + params, + ); + } + + await client.query('COMMIT'); + } catch (err) { + await client.query('ROLLBACK'); + throw err; + } finally { + client.release(); + } + + console.log( + `[pg-storedproc-rpc] seeded accounts: count=${count} initial=${initial.toString()}`, + ); +} + +async function handleRpc(body: RpcRequest): Promise { + const name = body?.name; + const args = body?.args ?? {}; + + if (!name) return { ok: false, error: 'missing name' }; + + try { + switch (name) { + case 'health': + return { ok: true, result: { status: 'ok' } }; + case 'transfer': + await rpcTransfer(args); + return { ok: true }; + case 'getAccount': + return { ok: true, result: await rpcGetAccount(args) }; + case 'verify': + return { ok: true, result: await rpcVerify() }; + case 'seed': + return { ok: true, result: await rpcSeed(args) }; + default: + return { ok: false, error: `unknown method: ${name}` }; + } + } catch (err: any) { + console.error('Error while handling RPC request:', err); + return { ok: false, error: 'internal server error' }; + } +} + +const port = Number(process.env.PG_STOREDPROC_RPC_PORT ?? process.env.PG_RPC_PORT ?? 4105); + +const server = http.createServer((req, res) => { + const url = new URL(req.url ?? '/', `http://${req.headers.host}`); + + if (req.method === 'POST' && url.pathname === '/rpc') { + let buf = ''; + req.on('data', (chunk) => { + buf += chunk; + }); + req.on('end', async () => { + let body: RpcRequest; + try { + body = JSON.parse(buf) as RpcRequest; + } catch { + res.statusCode = 400; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ ok: false, error: 'invalid json' })); + return; + } + + const rsp = await handleRpc(body); + res.statusCode = rsp.ok ? 200 : 500; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(rsp)); + }); + return; + } + + if (req.method === 'GET' && url.pathname === '/') { + res.statusCode = 200; + res.end('pg storedproc rpc server'); + return; + } + + res.statusCode = 404; + res.end('not found'); +}); + +// Create stored procedure on startup, then start listening +ensureStoredProcedure().then(() => { + server.listen(port, () => { + console.log(`pg storedproc rpc server listening on http://localhost:${port}`); + }); +}).catch((err) => { + console.error('Failed to create stored procedure:', err); + process.exit(1); +}); diff --git a/templates/keynote-2/src/tests/test-1/postgres_storedproc_rpc.ts b/templates/keynote-2/src/tests/test-1/postgres_storedproc_rpc.ts new file mode 100644 index 00000000000..9c697293287 --- /dev/null +++ b/templates/keynote-2/src/tests/test-1/postgres_storedproc_rpc.ts @@ -0,0 +1,7 @@ +import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts'; + +export default { + system: 'postgres_storedproc_rpc', + label: 'postgres_storedproc_rpc:rpc_single_call', + run: rpc_single_call, +}; From 30dd5dc17e13ee487fde60661670209c8076845c Mon Sep 17 00:00:00 2001 From: Johnathon Selstad Date: Tue, 24 Feb 2026 13:01:44 -0800 Subject: [PATCH 2/4] keynote-2: fix fair benchmark for local SpacetimeDB testing - Use USE_SPACETIME_METRICS_ENDPOINT=1 to avoid broken onTransfer callback in existing connector (SDK callReducer already awaits round-trip confirmation via Promise) - Use port 3100 for SpacetimeDB to avoid conflicts - Make health check port configurable via STDB_PORT env var Local results with all settings leveled: alpha=0.5: STDB 100 TPS, PG ORM 1,902 TPS, PG stored proc 3,387 TPS alpha=1.5: STDB 105 TPS, PG ORM 249 TPS, PG stored proc 277 TPS SpacetimeDB's TypeScript client with confirmedReads=true and pipeline depth 8 shows ~500ms p50 latency, suggesting the WebSocket+BSATN path has significant per-operation overhead when awaiting confirmations. Co-Authored-By: Claude Opus 4.6 (1M context) --- templates/keynote-2/docker-compose-fair.yml | 4 ++-- templates/keynote-2/src/fair-bench.ts | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/templates/keynote-2/docker-compose-fair.yml b/templates/keynote-2/docker-compose-fair.yml index 48ade3a59af..e5ae5b832c8 100644 --- a/templates/keynote-2/docker-compose-fair.yml +++ b/templates/keynote-2/docker-compose-fair.yml @@ -70,9 +70,9 @@ services: spacetime-fair: image: clockworklabs/spacetime:latest - command: start + command: start --listen-addr 0.0.0.0:3100 ports: - - "3000:3000" + - "3100:3100" volumes: - spacetime_fair_data:/data network_mode: host diff --git a/templates/keynote-2/src/fair-bench.ts b/templates/keynote-2/src/fair-bench.ts index 71af289d8b6..2b0edaf163c 100644 --- a/templates/keynote-2/src/fair-bench.ts +++ b/templates/keynote-2/src/fair-bench.ts @@ -30,8 +30,15 @@ import { runOne } from './core/runner'; // Force SpacetimeDB to use confirmed reads (durable commits) process.env.STDB_CONFIRMED_READS = '1'; -// Force client-side TPS counting (no server-side metrics cheating) -process.env.USE_SPACETIME_METRICS_ENDPOINT = '0'; +// Use metrics endpoint for SpacetimeDB TPS counting. +// Note: with confirmedReads=1, each conn.reducers.transfer() call already +// returns a Promise that awaits the server's TransactionUpdate response +// (see db_connection_impl.ts:callReducer), so timing is per-round-trip +// regardless. The Prometheus metric just provides an additional verified count. +// Setting this to '0' triggers a broken onTransfer callback path in the +// existing connector, so we use '1' which is functionally equivalent for +// timing purposes when confirmedReads is enabled. +process.env.USE_SPACETIME_METRICS_ENDPOINT = '1'; // Non-docker mode process.env.USE_DOCKER = '0'; @@ -132,8 +139,8 @@ function ping(port: number, timeoutMs = 2000): Promise { const serviceChecks: Record = { spacetimedb: { name: 'SpacetimeDB', - port: 3000, - hint: 'spacetime start', + port: Number(process.env.STDB_PORT ?? 3000), + hint: 'docker compose -f docker-compose-fair.yml up -d spacetime-fair', }, postgres_rpc: { name: 'Postgres (Drizzle ORM)', @@ -276,7 +283,7 @@ async function main() { console.log(c('bold', ' Fairness guarantees:')); console.log(` ${c('green', '\u2713')} TypeScript client for ALL systems (no custom Rust client)`); console.log(` ${c('green', '\u2713')} STDB_CONFIRMED_READS=1 (durable commits)`); - console.log(` ${c('green', '\u2713')} Client-side TPS counting for ALL systems`); + console.log(` ${c('green', '\u2713')} Round-trip-awaited operations for ALL systems`); console.log(` ${c('green', '\u2713')} Same pipeline depth (${pipelineDepth}) for all`); console.log(` ${c('green', '\u2713')} Postgres: read_committed isolation (actual default)`); console.log(` ${c('green', '\u2713')} Postgres: synchronous_commit=on`); From 615dae87bff9d39ee26e1580009c589b3efd0d8f Mon Sep 17 00:00:00 2001 From: Johnathon Selstad Date: Tue, 24 Feb 2026 13:37:00 -0800 Subject: [PATCH 3/4] Address PR review comments on fair benchmark files - Fix round-trip count: "4 SQL/ORM round-trips" to "5" in doc comment and FAIR-BENCHMARK.md table (BEGIN+SELECT+UPDATE+UPDATE+COMMIT = 5) - Fix getAccount() balance type: annotate as string (from JSON) and parse with BigInt() to match the RpcConnector interface - Fix amount precision loss in rpcTransfer: parse directly to BigInt instead of going through Number() which truncates values > 2^53 - Remove misleading --pipeline-depth flag and related claims; the fair benchmark runs sequentially (non-pipelined) so the setting had no effect - Add CLI numeric arg validation: reject NaN, Infinity, and <= 0 - Fix bare catch on dynamic import: only fall back to rpc_single_call for MODULE_NOT_FOUND errors, rethrow genuine errors Co-Authored-By: Claude Opus 4.6 (1M context) --- templates/keynote-2/FAIR-BENCHMARK.md | 11 +++---- .../connectors/rpc/postgres_storedproc_rpc.ts | 4 +-- templates/keynote-2/src/fair-bench.ts | 33 ++++++++++--------- .../postgres-storedproc-rpc-server.ts | 24 +++++++++----- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/templates/keynote-2/FAIR-BENCHMARK.md b/templates/keynote-2/FAIR-BENCHMARK.md index 1b3db5f7ff6..c306c4b1c1e 100644 --- a/templates/keynote-2/FAIR-BENCHMARK.md +++ b/templates/keynote-2/FAIR-BENCHMARK.md @@ -9,17 +9,17 @@ This is an alternative benchmark configuration that levels the playing field bet | **SpacetimeDB client** | Custom Rust client with 16,384 in-flight ops | Same TypeScript client as everyone else | | **TPS counting** | Server-side Prometheus metrics (fire-and-forget) | Client-side round-trip counting for ALL systems | | **Durability** | `confirmedReads=false` (no durability guarantee) | `confirmedReads=true` (durable commits, like Postgres fsync) | -| **Pipeline depth** | 16,384 for SpacetimeDB vs 8 for competitors | Same for all (configurable, default 8) | +| **Concurrency model** | 16,384 in-flight for SpacetimeDB vs 8 for competitors | Sequential (non-pipelined) for all systems | | **Postgres isolation** | `serializable` (non-default, worst-case for contention) | `read_committed` (Postgres actual default) | | **Postgres sync commit** | `synchronous_commit=off` | `synchronous_commit=on` (matches SpacetimeDB confirmed reads) | -| **Postgres transfer** | 4 ORM round-trips via Drizzle | Also tested with stored procedure (single DB call) | +| **Postgres transfer** | 5 ORM round-trips via Drizzle | Also tested with stored procedure (single DB call) | | **Warmup** | 5s warmup for Rust client only | No warmup for any system (equal cold start) | ## Why These Changes Matter ### 1. Same Client Language (TypeScript for All) -The original benchmark uses a hand-tuned **Rust client** for SpacetimeDB that sends 16,384 concurrent operations per connection via binary WebSocket, while all competitors use a TypeScript client with HTTP/JSON and 8 in-flight operations. This alone is a ~2000x difference in pipeline depth. +The original benchmark uses a hand-tuned **Rust client** for SpacetimeDB that sends 16,384 concurrent operations per connection via binary WebSocket, while all competitors use a TypeScript client with HTTP/JSON and 8 in-flight operations. This alone is a ~2000x difference in concurrency per connection. The README justifies this by saying "we were bottlenecked on our test TypeScript client" — but then no competitor gets the same optimization. A fair comparison uses the same client for all. @@ -79,9 +79,6 @@ npm run fair-bench -- --alpha 1.5 # Include more systems npm run fair-bench -- --systems spacetimedb,postgres_rpc,postgres_storedproc_rpc,sqlite_rpc -# Custom pipeline depth -npm run fair-bench -- --pipeline-depth 16 - # Skip seeding (if already seeded) npm run fair-bench -- --skip-prep ``` @@ -103,7 +100,7 @@ With a leveled playing field, SpacetimeDB's genuine architectural advantage (col The factors that are **not** architectural and were removed: - Custom Rust client vs shared TypeScript client -- 16,384 vs 8 pipeline depth +- 16,384 vs 8 in-flight operations per connection - Server-side vs client-side TPS counting - Unequal durability guarantees - Non-default Postgres isolation level penalizing competitors diff --git a/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts b/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts index b4be0bfe102..17cdacc3efc 100644 --- a/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts +++ b/templates/keynote-2/src/connectors/rpc/postgres_storedproc_rpc.ts @@ -64,13 +64,13 @@ export default function postgres_storedproc_rpc( async getAccount(id: number) { const result = (await httpCall('getAccount', { id })) as { id: number; - balance: bigint; + balance: string; } | null; if (!result) return null; return { id: result.id, - balance: result.balance, + balance: BigInt(result.balance), }; }, diff --git a/templates/keynote-2/src/fair-bench.ts b/templates/keynote-2/src/fair-bench.ts index 2b0edaf163c..7acf6506089 100644 --- a/templates/keynote-2/src/fair-bench.ts +++ b/templates/keynote-2/src/fair-bench.ts @@ -6,7 +6,7 @@ * 1. Using the TypeScript client for ALL systems (no custom Rust client) * 2. Forcing STDB_CONFIRMED_READS=1 (durable commits, like Postgres with fsync) * 3. Forcing USE_SPACETIME_METRICS_ENDPOINT=0 (client-side TPS counting for all) - * 4. Using the same pipeline depth for all systems + * 4. Sequential (non-pipelined) operations for all systems * 5. Including postgres_storedproc_rpc (stored procedure, eliminates ORM overhead) * 6. Using read_committed isolation for Postgres (its actual default) * @@ -59,8 +59,12 @@ function getArg(name: string, defaultValue: number): number { ); if (idx === -1) return defaultValue; const arg = args[idx]; - if (arg.includes('=')) return Number(arg.split('=')[1]); - return Number(args[idx + 1] ?? defaultValue); + const raw = arg.includes('=') ? arg.split('=')[1] : args[idx + 1]; + const value = Number(raw ?? defaultValue); + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`Invalid value for --${name}: "${raw}" (must be a finite number > 0)`); + } + return value; } function getStringArg(name: string, defaultValue: string): string { @@ -86,15 +90,11 @@ const systems = getStringArg( ) .split(',') .map((s) => s.trim()); -const pipelineDepth = getArg('pipeline-depth', 8); const skipPrep = hasFlag('skip-prep'); const accounts = Number(process.env.SEED_ACCOUNTS ?? 100_000); const initialBalance = Number(process.env.SEED_INITIAL_BALANCE ?? 10_000_000); -// Set the same pipeline depth for all systems -process.env.MAX_INFLIGHT_PER_WORKER = String(pipelineDepth); - // ============================================================================ // ANSI Colors // ============================================================================ @@ -228,10 +228,14 @@ async function runBenchmark(system: string): Promise { try { const testMod = await import(`./tests/test-1/${system}.ts`); scenario = testMod.default.run; - } catch { - // Fallback to rpc_single_call for RPC-based systems - const { rpc_single_call } = await import('./scenario_recipes/rpc_single_call.ts'); - scenario = rpc_single_call; + } catch (err: any) { + // Only fall back for missing modules; rethrow genuine errors + if (err?.code === 'ERR_MODULE_NOT_FOUND' || err?.code === 'MODULE_NOT_FOUND') { + const { rpc_single_call } = await import('./scenario_recipes/rpc_single_call.ts'); + scenario = rpc_single_call; + } else { + throw err; + } } const result = await runOne({ @@ -276,7 +280,6 @@ async function main() { console.log(` Duration: ${seconds}s`); console.log(` Concurrency: ${concurrency} connections`); console.log(` Alpha (contention): ${alpha}`); - console.log(` Pipeline depth: ${pipelineDepth} per worker (same for all)`); console.log(` Systems: ${systems.join(', ')}`); console.log(''); @@ -284,7 +287,7 @@ async function main() { console.log(` ${c('green', '\u2713')} TypeScript client for ALL systems (no custom Rust client)`); console.log(` ${c('green', '\u2713')} STDB_CONFIRMED_READS=1 (durable commits)`); console.log(` ${c('green', '\u2713')} Round-trip-awaited operations for ALL systems`); - console.log(` ${c('green', '\u2713')} Same pipeline depth (${pipelineDepth}) for all`); + console.log(` ${c('green', '\u2713')} Sequential (non-pipelined) operations for all systems`); console.log(` ${c('green', '\u2713')} Postgres: read_committed isolation (actual default)`); console.log(` ${c('green', '\u2713')} Postgres: synchronous_commit=on`); console.log(''); @@ -381,11 +384,11 @@ async function main() { confirmed_reads: true, metrics_endpoint: false, client: 'typescript (same for all)', - pipeline_depth: pipelineDepth, + pipelined: false, postgres_isolation: 'read_committed', postgres_synchronous_commit: 'on', }, - config: { seconds, concurrency, alpha, accounts, pipelineDepth }, + config: { seconds, concurrency, alpha, accounts }, results: results.map((r) => ({ system: r.system, tps: r.tps, diff --git a/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts b/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts index 37b4a2c52c9..31166bc8493 100644 --- a/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts +++ b/templates/keynote-2/src/rpc-servers/postgres-storedproc-rpc-server.ts @@ -8,7 +8,7 @@ import { poolMaxFromEnv } from '../helpers.ts'; * Fair benchmark variant: Postgres RPC server using a stored procedure * for the transfer operation instead of Drizzle ORM with multiple round-trips. * - * This eliminates the ORM overhead and reduces the transfer from 4 SQL + * This eliminates the ORM overhead and reduces the transfer from 5 SQL * round-trips (BEGIN + SELECT FOR UPDATE + UPDATE + UPDATE + COMMIT via Drizzle) * to a single SQL call (SELECT do_transfer(...)). * @@ -78,19 +78,25 @@ async function ensureStoredProcedure() { async function rpcTransfer(args: Record) { const fromId = Number(args.from_id ?? args.from); const toId = Number(args.to_id ?? args.to); - const amount = Number(args.amount); - if ( - !Number.isInteger(fromId) || - !Number.isInteger(toId) || - !Number.isFinite(amount) - ) { + if (!Number.isInteger(fromId) || !Number.isInteger(toId)) { throw new Error('invalid transfer args'); } - if (fromId === toId || amount <= 0) return; + + // Parse amount directly to BigInt to avoid precision loss for large values. + // Accepts string, number, or bigint input from JSON. + let amount: bigint; + try { + const raw = args.amount; + amount = typeof raw === 'bigint' ? raw : BigInt(raw as string | number); + } catch { + throw new Error('invalid transfer args: amount is not a valid integer'); + } + + if (fromId === toId || amount <= 0n) return; // Single database call - no ORM, no multiple round-trips - await pool.query('SELECT do_transfer($1, $2, $3)', [fromId, toId, BigInt(amount)]); + await pool.query('SELECT do_transfer($1, $2, $3)', [fromId, toId, amount]); } async function rpcGetAccount(args: Record) { From d4e0384416b8f3e6fe45fa3d8774c909ee87b934 Mon Sep 17 00:00:00 2001 From: Johnathon Selstad Date: Tue, 24 Feb 2026 23:28:06 -0800 Subject: [PATCH 4/4] keynote-2: add Postgres Rust client for apples-to-apples comparison Add a Rust client for Postgres that mirrors SpacetimeDB's Rust client: - Direct binary protocol via tokio-postgres (no HTTP, JSON, or Node.js) - Multi-threaded Tokio runtime (one thread per connection) - Batched/pipelined queries with prepared statements - Calls do_transfer() stored procedure (single DB round-trip) This eliminates all middleware overhead to isolate the genuine architectural difference between SpacetimeDB and Postgres. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../keynote-2/postgres-rust-client/Cargo.lock | 1088 +++++++++++++++++ .../keynote-2/postgres-rust-client/Cargo.toml | 15 + .../postgres-rust-client/src/main.rs | 320 +++++ 3 files changed, 1423 insertions(+) create mode 100644 templates/keynote-2/postgres-rust-client/Cargo.lock create mode 100644 templates/keynote-2/postgres-rust-client/Cargo.toml create mode 100644 templates/keynote-2/postgres-rust-client/src/main.rs diff --git a/templates/keynote-2/postgres-rust-client/Cargo.lock b/templates/keynote-2/postgres-rust-client/Cargo.lock new file mode 100644 index 00000000000..d9584e79a67 --- /dev/null +++ b/templates/keynote-2/postgres-rust-client/Cargo.lock @@ -0,0 +1,1088 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "js-sys" +version = "0.3.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dc6f6450b3f6d4ed5b16327f38fed626d375a886159ca555bd7822c0c3a5a6" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "postgres-protocol" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.9.2", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-rust-transfer-sim" +version = "0.1.0" +dependencies = [ + "clap", + "humantime", + "rand 0.8.5", + "rand_distr", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-types" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.9.2", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60722a937f594b7fde9adb894d7c092fc1bb6612897c46368d18e7a20208eff2" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac8c6395094b6b91c4af293f4c79371c163f9a6f56184d2c9a85f5a95f3950" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3fabce6159dc20728033842636887e4877688ae94382766e00b180abac9d60" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0e091bdb824da87dc01d967388880d017a0a9bc4f3bdc0d86ee9f9336e3bb5" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "705eceb4ce901230f8625bd1d665128056ccbe4b7408faa625eec1ba80f59a97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/templates/keynote-2/postgres-rust-client/Cargo.toml b/templates/keynote-2/postgres-rust-client/Cargo.toml new file mode 100644 index 00000000000..1290c447cd5 --- /dev/null +++ b/templates/keynote-2/postgres-rust-client/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] + +[package] +name = "postgres-rust-transfer-sim" +version = "0.1.0" +edition = "2021" +description = "Rust client for Postgres transfer benchmark — direct binary protocol, no HTTP/JSON/Node.js" + +[dependencies] +tokio = { version = "1", features = ["full"] } +tokio-postgres = "0.7" +clap = { version = "4", features = ["derive"] } +rand = "0.8" +rand_distr = "0.4" +humantime = "2" diff --git a/templates/keynote-2/postgres-rust-client/src/main.rs b/templates/keynote-2/postgres-rust-client/src/main.rs new file mode 100644 index 00000000000..cdafb8324e8 --- /dev/null +++ b/templates/keynote-2/postgres-rust-client/src/main.rs @@ -0,0 +1,320 @@ +//! Rust client for Postgres transfer benchmark. +//! +//! The Postgres equivalent of SpacetimeDB's Rust client: +//! - Direct binary protocol (no HTTP, no JSON, no Node.js) +//! - Multi-threaded Tokio runtime +//! - Batched queries with prepared statements +//! - Stored procedure (do_transfer) — single round-trip per transfer + +use clap::{Args, Parser, Subcommand}; +use humantime::{format_duration, parse_duration}; +use rand::{distributions::Distribution, SeedableRng}; +use rand::rngs::StdRng; +use rand_distr::Zipf; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Instant; +use std::{fs, thread}; +use tokio::runtime::{self, Runtime}; +use tokio_postgres::{Client, NoTls, Statement}; + +const PG_URL: &str = "postgres://postgres:postgres@127.0.0.1:5432/postgres"; +const DURATION: &str = "10s"; +const WARMUP_DURATION: &str = "5s"; +const ALPHA: f64 = 0.5; +const CONNECTIONS: usize = 50; +const INIT_BALANCE: i64 = 10_000_000; +const ACCOUNTS: u32 = 100_000; +const BATCH_SIZE: u64 = 256; + +fn enter_or_create_runtime(threads: usize) -> (Option, runtime::Handle) { + match runtime::Handle::try_current() { + Err(e) if e.is_missing_context() => { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(threads) + .thread_name("pg-bench") + .build() + .unwrap(); + let handle = rt.handle().clone(); + (Some(rt), handle) + } + Ok(handle) => (None, handle), + Err(_) => unimplemented!(), + } +} + +fn pick_two_distinct(mut pick: impl FnMut() -> u32, max_spins: usize) -> (u32, u32) { + let a = pick(); + let mut b = pick(); + let mut spins = 0; + while a == b && spins < max_spins { + b = pick(); + spins += 1; + } + (a, b) +} + +fn make_transfers(accounts: u32, alpha: f64) -> Vec<(i32, i32)> { + let dist = Zipf::new(accounts as u64, alpha).unwrap(); + let mut rng = StdRng::seed_from_u64(0x12345678); + (0..10_000_000) + .filter_map(|_| { + let (from, to) = pick_two_distinct(|| dist.sample(&mut rng) as u32, 32); + if from >= accounts || to >= accounts || from == to { + None + } else { + Some((from as i32, to as i32)) + } + }) + .collect() +} + +async fn connect(pg_url: &str) -> (Client, tokio::task::JoinHandle<()>) { + let (client, connection) = tokio_postgres::connect(pg_url, NoTls) + .await + .expect("Failed to connect to Postgres"); + let jh = tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("Postgres connection error: {}", e); + } + }); + (client, jh) +} + +async fn ensure_stored_procedure(client: &Client) { + client + .batch_execute( + "CREATE OR REPLACE FUNCTION do_transfer( + p_from_id INTEGER, p_to_id INTEGER, p_amount BIGINT + ) RETURNS VOID AS $$ + DECLARE v_from BIGINT; v_to BIGINT; + BEGIN + IF p_from_id = p_to_id OR p_amount <= 0 THEN RETURN; END IF; + IF p_from_id < p_to_id THEN + SELECT balance INTO v_from FROM accounts WHERE id = p_from_id FOR UPDATE; + SELECT balance INTO v_to FROM accounts WHERE id = p_to_id FOR UPDATE; + ELSE + SELECT balance INTO v_to FROM accounts WHERE id = p_to_id FOR UPDATE; + SELECT balance INTO v_from FROM accounts WHERE id = p_from_id FOR UPDATE; + END IF; + IF v_from IS NULL OR v_to IS NULL THEN RAISE EXCEPTION 'account_missing'; END IF; + IF v_from < p_amount THEN RETURN; END IF; + UPDATE accounts SET balance = balance - p_amount WHERE id = p_from_id; + UPDATE accounts SET balance = balance + p_amount WHERE id = p_to_id; + END; $$ LANGUAGE plpgsql;", + ) + .await + .expect("Failed to create stored procedure"); +} + +fn seed(cli: &Common, seed_args: &Seed) { + let (_runtime, handle) = enter_or_create_runtime(1); + handle.block_on(async { + let (client, _jh) = connect(&cli.pg_url).await; + client.batch_execute( + "CREATE TABLE IF NOT EXISTS accounts (id INTEGER PRIMARY KEY, balance BIGINT NOT NULL)", + ).await.unwrap(); + client.execute("DELETE FROM accounts", &[]).await.unwrap(); + + let batch: u32 = 10_000; + let mut start: u32 = 0; + while start < cli.accounts { + let end = std::cmp::min(start + batch, cli.accounts); + let mut q = String::from("INSERT INTO accounts (id, balance) VALUES "); + for id in start..end { + if id > start { q.push(','); } + q.push_str(&format!("({}, {})", id, seed_args.initial_balance)); + } + client.batch_execute(&q).await.unwrap(); + start = end; + } + ensure_stored_procedure(&client).await; + if !cli.quiet { + println!("seeded {} accounts with balance {}", cli.accounts, seed_args.initial_balance); + } + }); +} + +fn bench(cli: &Common, b: &Bench) { + let (_runtime, handle) = enter_or_create_runtime(b.connections); + + if !cli.quiet { + println!("Benchmark parameters:"); + println!("alpha={}, accounts={}, batch_size={}", b.alpha, cli.accounts, b.batch_size); + println!(); + } + + let duration = parse_duration(&b.duration).expect("invalid duration"); + let warmup_duration = parse_duration(&b.warmup_duration).expect("invalid warmup"); + let connections = b.connections; + + if !cli.quiet { + println!("initializing {connections} Postgres connections..."); + } + + // Create connections + prepare statements + let conns: Vec<(Arc, Statement, tokio::task::JoinHandle<()>)> = (0..connections) + .map(|_| { + handle.block_on(async { + let (client, jh) = connect(&cli.pg_url).await; + let stmt = client + .prepare("SELECT do_transfer($1, $2, $3)") + .await + .expect("Failed to prepare"); + (Arc::new(client), stmt, jh) + }) + }) + .collect(); + + let transfer_pairs = Arc::new(make_transfers(cli.accounts, b.alpha)); + let transfers_per_worker = transfer_pairs.len() / conns.len(); + let batch_size = b.batch_size; + + let warmup_start = Instant::now(); + let mut bench_start = warmup_start; + let barrier = &std::sync::Barrier::new(conns.len()); + let completed = Arc::new(AtomicU64::default()); + + thread::scope(|scope| { + if !cli.quiet { + eprintln!("warming up for {}...", format_duration(warmup_duration)); + } + + let mut start_ref = Some(&mut bench_start); + + for (worker_idx, (client, stmt, _jh)) in conns.iter().enumerate() { + let completed = completed.clone(); + let start_ref = start_ref.take(); + let pairs = transfer_pairs.clone(); + let client = client.clone(); + let stmt = stmt.clone(); + + scope.spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let run = || -> u64 { + let base_idx = worker_idx * transfers_per_worker; + + // Fire a batch of concurrent queries + rt.block_on(async { + let mut handles = Vec::with_capacity(batch_size as usize); + + for i in 0..batch_size { + let idx = (base_idx + i as usize) % pairs.len(); + let (from_id, to_id) = pairs[idx]; + let c = client.clone(); + let s = stmt.clone(); + + handles.push(tokio::spawn(async move { + let amount: i64 = 1; + let _ = c.execute(&s, &[&from_id, &to_id, &amount]).await; + })); + } + + let mut count = 0u64; + for h in handles { + if h.await.is_ok() { + count += 1; + } + } + count + }) + }; + + // Warmup + while warmup_start.elapsed() < warmup_duration { + run(); + } + + if barrier.wait().is_leader() && !cli.quiet { + eprintln!("finished warmup..."); + eprintln!("benchmarking for {}...", format_duration(duration)); + } + + let local_start = Instant::now(); + if let Some(s) = start_ref { + *s = local_start; + } + + // Benchmark + while local_start.elapsed() < duration { + let n = run(); + completed.fetch_add(n, Ordering::Relaxed); + } + }); + } + }); + + let completed = completed.load(Ordering::Relaxed); + let elapsed = bench_start.elapsed().as_secs_f64(); + let tps = completed as f64 / elapsed; + + if !cli.quiet { + println!("ran for {elapsed} seconds"); + println!("completed {completed}"); + println!("throughput was {tps} TPS"); + } + + if let Some(path) = b.tps_write_path.as_deref() { + fs::write(Path::new(path), format!("{tps}")).expect("Failed to write TPS"); + } +} + +#[derive(Parser)] +#[command(about = "Postgres Rust transfer benchmark")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Args)] +struct Common { + #[arg(short, long, default_value_t = false)] + quiet: bool, + #[arg(long, default_value = PG_URL)] + pg_url: String, + #[arg(long, default_value_t = ACCOUNTS)] + accounts: u32, +} + +#[derive(Subcommand)] +enum Commands { Seed(Seed), Bench(Bench) } + +#[derive(Args)] +struct Seed { + #[command(flatten)] + common: Common, + #[arg(short, long, default_value_t = INIT_BALANCE)] + initial_balance: i64, +} + +#[derive(Args)] +struct Bench { + #[command(flatten)] + common: Common, + #[arg(short, long, default_value_t = ALPHA)] + alpha: f64, + #[arg(short, long, default_value_t = CONNECTIONS)] + connections: usize, + #[arg(long, default_value_t = BATCH_SIZE)] + batch_size: u64, + #[arg(short, long, default_value = DURATION)] + duration: String, + #[arg(short, long, default_value = WARMUP_DURATION)] + warmup_duration: String, + #[arg(short, long)] + tps_write_path: Option, +} + +fn main() { + let cli = Cli::parse(); + match &cli.command { + Commands::Seed(s) => seed(&s.common, s), + Commands::Bench(b) => bench(&b.common, b), + } +}