diff --git a/templates/keynote-2/postgres-go-client/.gitignore b/templates/keynote-2/postgres-go-client/.gitignore new file mode 100644 index 00000000000..52781ec519d --- /dev/null +++ b/templates/keynote-2/postgres-go-client/.gitignore @@ -0,0 +1,3 @@ +# Ignore the log and output of the client +log +tps diff --git a/templates/keynote-2/postgres-go-client/go.mod b/templates/keynote-2/postgres-go-client/go.mod new file mode 100644 index 00000000000..8b864dc4419 --- /dev/null +++ b/templates/keynote-2/postgres-go-client/go.mod @@ -0,0 +1,5 @@ +module benchmark + +go 1.25.0 + +require github.com/lib/pq v1.11.2 // indirect diff --git a/templates/keynote-2/postgres-go-client/go.sum b/templates/keynote-2/postgres-go-client/go.sum new file mode 100644 index 00000000000..de0b64ebf36 --- /dev/null +++ b/templates/keynote-2/postgres-go-client/go.sum @@ -0,0 +1,2 @@ +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= diff --git a/templates/keynote-2/postgres-go-client/main.go b/templates/keynote-2/postgres-go-client/main.go new file mode 100644 index 00000000000..e91240bfc12 --- /dev/null +++ b/templates/keynote-2/postgres-go-client/main.go @@ -0,0 +1,262 @@ +// main.go +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "math" + "math/rand" + "os" + "sync" + "sync/atomic" + "time" + + _ "github.com/lib/pq" +) + +const ( + defaultPGURL = "postgres://localhost:5432/postgres?sslmode=disable" + defaultDuration = 10 * time.Second + defaultWarmupDuration = 5 * time.Second + defaultAlpha = 0.5 + defaultConnections = 10 + defaultAmount = 1 + defaultAccounts = 100_000 + defaultMaxInflight = 128 +) + +type BenchConfig struct { + PGURL string + Accounts uint32 + Alpha float64 + Amount uint32 + Connections int + MaxInflight int + Duration time.Duration + WarmupDuration time.Duration + TPSWritePath string + Quiet bool +} + +// zipfSample returns a Zipf-distributed value in [1, n]. +func zipfSample(rng *rand.Rand, n uint32, alpha float64) uint32 { + // Rejection-based Zipf sampling + // Using Go's built-in Zipf generator + z := rand.NewZipf(rng, alpha+1.0, 1.0, uint64(n)-1) + return uint32(z.Uint64()) + 1 +} + +func pickTwoDistinct(rng *rand.Rand, accounts uint32, alpha float64) (uint32, uint32) { + a := zipfSample(rng, accounts, alpha) + b := zipfSample(rng, accounts, alpha) + for spins := 0; a == b && spins < 32; spins++ { + b = zipfSample(rng, accounts, alpha) + } + return a, b +} + +func makeTransfers(accounts uint32, alpha float64) [][2]uint32 { + rng := rand.New(rand.NewSource(0x12345678)) + pairs := make([][2]uint32, 0, 10_000_000) + for i := 0; i < 10_000_000; i++ { + from, to := pickTwoDistinct(rng, accounts, alpha) + if from >= accounts || to >= accounts || from == to { + continue + } + pairs = append(pairs, [2]uint32{from, to}) + } + return pairs +} + +func openPool(pgURL string, maxConns int) (*sql.DB, error) { + db, err := sql.Open("postgres", pgURL) + if err != nil { + return nil, fmt.Errorf("failed to open db: %w", err) + } + db.SetMaxOpenConns(maxConns) + db.SetMaxIdleConns(maxConns) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := db.PingContext(ctx); err != nil { + return nil, fmt.Errorf("failed to ping db: %w", err) + } + return db, nil +} + +func runBench(cfg BenchConfig) error { + if !cfg.Quiet { + fmt.Println("Benchmark parameters:") + fmt.Printf(" alpha=%.2f, amount=%d, accounts=%d\n", cfg.Alpha, cfg.Amount, cfg.Accounts) + fmt.Printf(" connections=%d, max_inflight_per_conn=%d\n", cfg.Connections, cfg.MaxInflight) + fmt.Printf(" warmup=%s, duration=%s\n", cfg.WarmupDuration, cfg.Duration) + fmt.Println() + } + + db, err := openPool(cfg.PGURL, cfg.Connections) + if err != nil { + return err + } + defer db.Close() + + // Pre-compute transfer pairs. + if !cfg.Quiet { + fmt.Println("pre-computing transfer pairs...") + } + transferPairs := makeTransfers(cfg.Accounts, cfg.Alpha) + pairsPerWorker := len(transferPairs) / cfg.Connections + if pairsPerWorker == 0 { + pairsPerWorker = len(transferPairs) + } + + if !cfg.Quiet { + fmt.Printf("generated %d transfer pairs (%d per worker)\n\n", len(transferPairs), pairsPerWorker) + } + + // Acquire one connection per worker up front. + conns := make([]*sql.Conn, cfg.Connections) + for i := 0; i < cfg.Connections; i++ { + c, err := db.Conn(context.Background()) + if err != nil { + return fmt.Errorf("failed to acquire connection %d: %w", i, err) + } + conns[i] = c + } + defer func() { + for _, c := range conns { + c.Close() + } + }() + + query := "CALL transfer($1, $2, $3)" + + // runBatch executes up to maxInflight sequential transfers and returns the count. + runBatch := func(conn *sql.Conn, pairs [][2]uint32, idx *int, max int) (int, error) { + count := 0 + for count < max { + if *idx >= len(pairs) { + *idx = 0 + } + p := pairs[*idx] + *idx++ + _, err := conn.ExecContext(context.Background(), query, p[0], p[1], cfg.Amount) + if err != nil { + // Insufficient funds errors are expected; skip them. + count++ + continue + } + count++ + } + return count, nil + } + + // --- Warmup phase --- + if !cfg.Quiet { + fmt.Printf("warming up for %s...\n", cfg.WarmupDuration) + } + + var warmupWg sync.WaitGroup + warmupDeadline := time.Now().Add(cfg.WarmupDuration) + for i := 0; i < cfg.Connections; i++ { + warmupWg.Add(1) + go func(workerIdx int) { + defer warmupWg.Done() + conn := conns[workerIdx] + startIdx := (workerIdx * pairsPerWorker) % len(transferPairs) + idx := startIdx + myPairs := transferPairs + for time.Now().Before(warmupDeadline) { + runBatch(conn, myPairs, &idx, cfg.MaxInflight) + } + }(i) + } + warmupWg.Wait() + + if !cfg.Quiet { + fmt.Println("finished warmup.") + fmt.Printf("benchmarking for %s...\n", cfg.Duration) + } + + // --- Benchmark phase --- + var completed atomic.Int64 + var benchWg sync.WaitGroup + benchStart := time.Now() + benchDeadline := benchStart.Add(cfg.Duration) + + for i := 0; i < cfg.Connections; i++ { + benchWg.Add(1) + go func(workerIdx int) { + defer benchWg.Done() + conn := conns[workerIdx] + startIdx := (workerIdx * pairsPerWorker) % len(transferPairs) + idx := startIdx + myPairs := transferPairs + for time.Now().Before(benchDeadline) { + n, _ := runBatch(conn, myPairs, &idx, cfg.MaxInflight) + completed.Add(int64(n)) + } + }(i) + } + benchWg.Wait() + + elapsed := time.Since(benchStart).Seconds() + total := completed.Load() + tps := float64(total) / elapsed + + if !cfg.Quiet { + fmt.Printf("\nran for %.3f seconds\n", elapsed) + fmt.Printf("completed %d transfers\n", total) + fmt.Printf("throughput: %.2f TPS\n", tps) + } + + if math.IsNaN(tps) || math.IsInf(tps, 0) { + tps = 0 + } + + if cfg.TPSWritePath != "" { + if err := os.WriteFile(cfg.TPSWritePath, []byte(fmt.Sprintf("%f", tps)), 0644); err != nil { + return fmt.Errorf("failed to write TPS file: %w", err) + } + } + + // Always print the raw TPS to stdout for scripting. + if cfg.Quiet { + fmt.Println(tps) + } + + return nil +} + +func main() { + cfg := BenchConfig{} + + var accounts, amount uint + flag.StringVar(&cfg.PGURL, "pg-url", defaultPGURL, "PostgreSQL connection URL (or PG_URL env)") + flag.UintVar(&accounts, "accounts", uint(defaultAccounts), "number of accounts") + flag.Float64Var(&cfg.Alpha, "alpha", defaultAlpha, "Zipf alpha parameter") + flag.UintVar(&amount, "amount", defaultAmount, "transfer amount") + flag.IntVar(&cfg.Connections, "connections", defaultConnections, "number of parallel connections") + flag.IntVar(&cfg.MaxInflight, "max-inflight", defaultMaxInflight, "max sequential transfers per batch") + flag.DurationVar(&cfg.Duration, "duration", defaultDuration, "benchmark duration") + flag.DurationVar(&cfg.WarmupDuration, "warmup-duration", defaultWarmupDuration, "warmup duration") + flag.StringVar(&cfg.TPSWritePath, "tps-write-path", "", "file path to write TPS result") + flag.BoolVar(&cfg.Quiet, "quiet", false, "suppress informational output") + flag.Parse() + + cfg.Accounts = uint32(accounts) + cfg.Amount = uint32(amount) + + if err := runBench(cfg); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func envOrDefault(key, fallback string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return fallback +} \ No newline at end of file diff --git a/templates/keynote-2/src/connectors/index.ts b/templates/keynote-2/src/connectors/index.ts index 14a57e4d360..b1f8e1122a3 100644 --- a/templates/keynote-2/src/connectors/index.ts +++ b/templates/keynote-2/src/connectors/index.ts @@ -1,6 +1,8 @@ import convex from './convex.ts'; import { spacetimedb } from './spacetimedb.ts'; import bun from './bun.ts'; +import { postgres } from './postgres.ts'; +import { postgres_no_rpc } from './postgres.ts'; import postgres_rpc from './rpc/postgres_rpc.ts'; import cockroach_rpc from './rpc/cockroach_rpc.ts'; import sqlite_rpc from './rpc/sqlite_rpc.ts'; @@ -11,6 +13,8 @@ export const CONNECTORS = { convex, spacetimedb, bun, + postgres, + postgres_no_rpc, postgres_rpc, cockroach_rpc, sqlite_rpc, diff --git a/templates/keynote-2/src/connectors/postgres.ts b/templates/keynote-2/src/connectors/postgres.ts new file mode 100644 index 00000000000..3a163768492 --- /dev/null +++ b/templates/keynote-2/src/connectors/postgres.ts @@ -0,0 +1,151 @@ +import type {RpcConnector} from '../core/connectors'; +import * as console from "node:console"; +import {Pool} from "pg"; + +export function postgres_no_rpc(url: string): RpcConnector { + return postgres(url); +} + +export function postgres( + url = process.env.PG_URL! +): RpcConnector { + // Ready or not? + let ready!: Promise; + let resolveReady!: () => void; + let rejectReady!: (e: unknown) => void; + + function armReady() { + ready = new Promise((res, rej) => { + resolveReady = res; + rejectReady = rej; + }); + } + + let pool: Pool; + + async function connect() { + if (!url) throw new Error('PG_URL not set'); + armReady() + pool = new Pool({ + connectionString: process.env.PG_URL, + }); + try { + await pool.query("SELECT 1"); + resolveReady() + } catch (err) { + let message = "Failed to connect to postgres"; + if (err instanceof Error) { + message += ": " + err.message; + } + rejectReady(new Error(message)); + } + } + + return { + name: 'postgres', + + async open() { + try { + await connect(); + await ready; + } catch (err) { + console.error('[postgres] open() failed', err); + throw err; + } + }, + + async close() { + try { + await pool.end(); + } catch (err) { + console.error('[postgres] close() failed', err); + } + }, + + async call(fn: string, args: Record) { + switch (fn) { + case 'seed': { + try { + await pool.query( + `DROP TABLE IF EXISTS account; + DROP INDEX IF EXISTS account_id_index; + CREATE TABLE account + ( + id INTEGER PRIMARY KEY, + balance BIGINT + ); + CREATE INDEX account_id_index ON account USING HASH (id);` + ); + await pool.query( + `CREATE OR REPLACE PROCEDURE seed(n INTEGER, balance BIGINT) + LANGUAGE plpgsql + AS $$ + BEGIN + DELETE FROM account; + INSERT INTO account (id, balance) + SELECT g, balance + FROM generate_series(0, n - 1) AS g; + END; + $$;` + ); + await pool.query( + `CREATE OR REPLACE PROCEDURE transfer(from_id INTEGER, to_id INTEGER, amount BIGINT) + LANGUAGE plpgsql + AS $$ + DECLARE + from_balance BIGINT; + to_balance BIGINT; + BEGIN + SELECT balance INTO STRICT from_balance + FROM account + WHERE id = from_id; + + IF from_balance < amount THEN + RAISE EXCEPTION 'insufficient_funds'; + END IF; + + SELECT balance INTO STRICT to_balance + FROM account + WHERE id = to_id; + + UPDATE account + SET balance = balance - amount + WHERE id = from_id; + + UPDATE account + SET balance = balance + amount + WHERE id = to_id; + END; + $$;` + ); + await pool.query(`CALL seed(${args.accounts}, ${args.initialBalance})`); + } catch (err) { + let message = "Failed to seed"; + if (err instanceof Error) { + message += ": " + err.message; + } + throw new Error(message); + } + return; + } + case 'transfer': { + // console.log("transfer: " + args.from + ", " + args.to + ", " + args.amount) + await pool.query(`CALL transfer(${args.from}, ${args.to}, ${args.amount})`); + return; + } + default: + throw new Error(`Unknown function: ${fn}`); + } + }, + + async getAccount(id: number): Promise<{ id: number; balance: bigint; } | null> { + console.log(id) + // TODO + return null; + }, + + async verify(): Promise { + return Promise.resolve(undefined); + }, + }; +} diff --git a/templates/keynote-2/src/demo.ts b/templates/keynote-2/src/demo.ts index 7a4a27e61d5..d42e7c9c381 100644 --- a/templates/keynote-2/src/demo.ts +++ b/templates/keynote-2/src/demo.ts @@ -1,42 +1,42 @@ import 'dotenv/config'; -import { execSync } from 'node:child_process'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { createConnection } from 'node:net'; -import { join } from 'node:path'; -import { CONNECTORS } from './connectors'; -import { runOne } from './core/runner'; -import { initConvex } from './init/init_convex'; -import { sh } from './init/utils'; +import {execSync} from 'node:child_process'; +import {mkdir, writeFile} from 'node:fs/promises'; +import {createConnection} from 'node:net'; +import {join} from 'node:path'; +import {CONNECTORS} from './connectors'; +import {runOne} from './core/runner'; +import {initConvex} from './init/init_convex'; +import {sh} from './init/utils'; import * as fs from 'fs'; // Simple TCP ping - just check if something is listening on the port function ping(port: number, timeoutMs = 2000): Promise { - return new Promise((resolve) => { - const socket = createConnection({ host: '127.0.0.1', port }); - const timer = setTimeout(() => { - socket.destroy(); - resolve(false); - }, timeoutMs); - socket.on('connect', () => { - clearTimeout(timer); - socket.destroy(); - resolve(true); + return new Promise((resolve) => { + const socket = createConnection({host: '127.0.0.1', port}); + const timer = setTimeout(() => { + socket.destroy(); + resolve(false); + }, timeoutMs); + socket.on('connect', () => { + clearTimeout(timer); + socket.destroy(); + resolve(true); + }); + socket.on('error', () => { + clearTimeout(timer); + resolve(false); + }); }); - socket.on('error', () => { - clearTimeout(timer); - resolve(false); - }); - }); } // Use spacetime CLI to ping the server function spacetimePing(): boolean { - try { - execSync('spacetime server ping local', { stdio: 'ignore' }); - return true; - } catch { - return false; - } + try { + execSync('spacetime server ping local', {stdio: 'ignore'}); + return true; + } catch { + return false; + } } // ============================================================================ @@ -46,35 +46,35 @@ function spacetimePing(): boolean { const args = process.argv.slice(2); function getArg(name: string, defaultValue: number): number { - const idx = args.findIndex( - (a) => a === `--${name}` || a.startsWith(`--${name}=`), - ); - if (idx === -1) return defaultValue; - const arg = args[idx]; - if (arg.includes('=')) return Number(arg.split('=')[1]); - return Number(args[idx + 1] ?? defaultValue); + const idx = args.findIndex( + (a) => a === `--${name}` || a.startsWith(`--${name}=`), + ); + if (idx === -1) return defaultValue; + const arg = args[idx]; + if (arg.includes('=')) return Number(arg.split('=')[1]); + return Number(args[idx + 1] ?? defaultValue); } function getStringArg(name: string, defaultValue: string): string { - const idx = args.findIndex( - (a) => a === `--${name}` || a.startsWith(`--${name}=`), - ); - if (idx === -1) return defaultValue; - const arg = args[idx]; - if (arg.includes('=')) return arg.split('=')[1]; - return args[idx + 1] ?? defaultValue; + const idx = args.findIndex( + (a) => a === `--${name}` || a.startsWith(`--${name}=`), + ); + if (idx === -1) return defaultValue; + const arg = args[idx]; + if (arg.includes('=')) return arg.split('=')[1]; + return args[idx + 1] ?? defaultValue; } function hasFlag(name: string): boolean { - return args.includes(`--${name}`); + return args.includes(`--${name}`); } const seconds = getArg('seconds', 10); const concurrency = getArg('concurrency', 10); const alpha = getArg('alpha', 0.5); const systems = getStringArg('systems', 'convex,spacetimedb') - .split(',') - .map((s) => s.trim()); + .split(',') + .map((s) => s.trim()); const skipPrep = hasFlag('skip-prep'); const noAnimation = hasFlag('no-animation'); @@ -94,50 +94,50 @@ if (!process.env.STDB_MODULE) process.env.STDB_MODULE = 'test-1'; // ============================================================================ const colors = { - reset: '\x1b[0m', - bold: '\x1b[1m', - dim: '\x1b[2m', - green: '\x1b[32m', - yellow: '\x1b[33m', - blue: '\x1b[34m', - cyan: '\x1b[36m', - red: '\x1b[31m', + reset: '\x1b[0m', + bold: '\x1b[1m', + dim: '\x1b[2m', + green: '\x1b[32m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + cyan: '\x1b[36m', + red: '\x1b[31m', }; function c(color: keyof typeof colors, text: string): string { - if (noAnimation) return text; - return `${colors[color]}${text}${colors.reset}`; + if (noAnimation) return text; + return `${colors[color]}${text}${colors.reset}`; } const spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; function createSpinner(label: string): { stop: (finalText: string) => void } { - if (noAnimation) { - process.stdout.write(` ${label}...`); - return { - stop: (finalText: string) => { - console.log(` ${finalText}`); - }, - }; - } + if (noAnimation) { + process.stdout.write(` ${label}...`); + return { + stop: (finalText: string) => { + console.log(` ${finalText}`); + }, + }; + } - let frame = 0; - const interval = setInterval(() => { - process.stdout.write( - `\r ${spinnerFrames[frame++ % spinnerFrames.length]} ${label}...`, - ); - }, 80); + let frame = 0; + const interval = setInterval(() => { + process.stdout.write( + `\r ${spinnerFrames[frame++ % spinnerFrames.length]} ${label}...`, + ); + }, 80); - return { - stop: (finalText: string) => { - clearInterval(interval); - process.stdout.write(`\r ${label}... ${finalText} \n`); - }, - }; + return { + stop: (finalText: string) => { + clearInterval(interval); + process.stdout.write(`\r ${label}... ${finalText} \n`); + }, + }; } function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } // ============================================================================ @@ -145,78 +145,88 @@ function sleep(ms: number): Promise { // ============================================================================ interface ServiceConfig { - name: string; - healthCheck: () => Promise; - startCmd: string; - startCwd?: string; + name: string; + healthCheck: () => Promise; + startCmd: string; + startCwd?: string; } const serviceConfigs: Record = { - spacetimedb: { - name: 'SpacetimeDB', - healthCheck: async () => spacetimePing(), - startCmd: 'spacetime start', - }, - convex: { - name: 'Convex', - healthCheck: () => ping(3210), - startCmd: 'npx convex dev', - startCwd: 'convex-app', - }, - postgres_rpc: { - name: 'Postgres RPC', - healthCheck: () => ping(4101), - startCmd: 'npx tsx src/rpc-servers/postgres-rpc-server.ts', - }, - sqlite_rpc: { - name: 'SQLite RPC', - healthCheck: () => ping(4103), - startCmd: 'npx tsx src/rpc-servers/sqlite-rpc-server.ts', - }, - cockroach_rpc: { - name: 'CockroachDB RPC', - healthCheck: () => ping(4102), - startCmd: 'npx tsx src/rpc-servers/cockroach-rpc-server.ts', - }, - supabase_rpc: { - name: 'Supabase RPC', - healthCheck: () => ping(4106), - startCmd: 'npx tsx src/rpc-servers/supabase-rpc-server.ts', - }, - bun: { - name: 'Bun', - healthCheck: () => ping(4001), - startCmd: 'bun run bun/bun-server.ts', - }, + spacetimedb: { + name: 'SpacetimeDB', + healthCheck: async () => spacetimePing(), + startCmd: 'spacetime start', + }, + convex: { + name: 'Convex', + healthCheck: () => ping(3210), + startCmd: 'npx convex dev', + startCwd: 'convex-app', + }, + postgres: { + name: 'Postgres', + healthCheck: () => ping(5432), + startCmd: 'Start postgres', + }, + postgres_no_rpc: { + name: 'Postgres NO RPC', + healthCheck: () => ping(5432), + startCmd: 'Start postgres', + }, + postgres_rpc: { + name: 'Postgres RPC', + healthCheck: () => ping(4101), + startCmd: 'npx tsx src/rpc-servers/postgres-rpc-server.ts', + }, + sqlite_rpc: { + name: 'SQLite RPC', + healthCheck: () => ping(4103), + startCmd: 'npx tsx src/rpc-servers/sqlite-rpc-server.ts', + }, + cockroach_rpc: { + name: 'CockroachDB RPC', + healthCheck: () => ping(4102), + startCmd: 'npx tsx src/rpc-servers/cockroach-rpc-server.ts', + }, + supabase_rpc: { + name: 'Supabase RPC', + healthCheck: () => ping(4106), + startCmd: 'npx tsx src/rpc-servers/supabase-rpc-server.ts', + }, + bun: { + name: 'Bun', + healthCheck: () => ping(4001), + startCmd: 'bun run bun/bun-server.ts', + }, }; async function checkService(system: string): Promise { - const config = serviceConfigs[system]; - if (!config) return true; // Unknown system, assume ready - - const isRunning = await config.healthCheck(); - if (isRunning) { - console.log(` ${config.name.padEnd(15)} ${c('green', '✓')}`); - return true; - } - - console.log(` ${config.name.padEnd(15)} ${c('red', '✗ NOT RUNNING')}`); - console.log(`\n Please start ${config.name} in another terminal:`); - console.log(` ${c('cyan', config.startCmd)}`); - if (config.startCwd) { - console.log(` ${c('dim', `(from directory: ${config.startCwd})`)}`); - } - console.log(`\n Press Enter when ready...`); - - await new Promise((resolve) => { - process.stdin.once('data', () => resolve()); - }); - - const nowRunning = await config.healthCheck(); - if (nowRunning) { - console.log(` ${config.name.padEnd(15)} ${c('green', '✓')}`); - } - return nowRunning; + const config = serviceConfigs[system]; + if (!config) return true; // Unknown system, assume ready + + const isRunning = await config.healthCheck(); + if (isRunning) { + console.log(` ${config.name.padEnd(15)} ${c('green', '✓')}`); + return true; + } + + console.log(` ${config.name.padEnd(15)} ${c('red', '✗ NOT RUNNING')}`); + console.log(`\n Please start ${config.name} in another terminal:`); + console.log(` ${c('cyan', config.startCmd)}`); + if (config.startCwd) { + console.log(` ${c('dim', `(from directory: ${config.startCwd})`)}`); + } + console.log(`\n Press Enter when ready...`); + + await new Promise((resolve) => { + process.stdin.once('data', () => resolve()); + }); + + const nowRunning = await config.healthCheck(); + if (nowRunning) { + console.log(` ${config.name.padEnd(15)} ${c('green', '✓')}`); + } + return nowRunning; } // ============================================================================ @@ -224,142 +234,169 @@ async function checkService(system: string): Promise { // ============================================================================ async function prepSystem(system: string): Promise { - const connector = (CONNECTORS as any)[system]; - if (!connector) { - console.log(` ${system.padEnd(15)} ${c('yellow', '⚠ SKIPPED')}`); - return; - } + const connector = (CONNECTORS as any)[system]; + if (!connector) { + console.log(` ${system.padEnd(15)} ${c('yellow', '⚠ SKIPPED')}`); + return; + } - const spinner = createSpinner(system.padEnd(15)); + const spinner = createSpinner(system.padEnd(15)); + + try { + if (system === 'spacetimedb') { + const moduleName = process.env.STDB_MODULE || 'test-1'; + const server = process.env.STDB_SERVER || 'local'; + const server2 = process.env.STDB_SERVER || 'http://localhost:3000'; + const modulePath = process.env.STDB_MODULE_PATH || './spacetimedb'; + + // Publish module (creates DB if needed, updates if exists) + await sh('spacetime', [ + 'publish', + '--server', + server, + moduleName, + '--module-path', + modulePath, + ]); + await sh('cargo', [ + 'run', + //"--quiet", + "--manifest-path", + "spacetimedb-rust-client/Cargo.toml", + "--", + "seed", + //"--quiet", + '--server', + server2, + "--module", + moduleName, + "--accounts", + String(accounts), + "--initial-balance", + String(initialBalance), + ]); + console.log('[spacetimedb] seed complete.'); + } else if (system === 'convex') { + await initConvex(); + } else { + const conn = connector(); + await conn.open(); + await conn.call('seed', {accounts, initialBalance}); + await conn.close(); + } + spinner.stop(c('green', '✓ READY')); + } catch (err: any) { + spinner.stop(c('red', `✗ ${err.message}`)); + } +} - try { - if (system === 'spacetimedb') { - const moduleName = process.env.STDB_MODULE || 'test-1'; - const server = process.env.STDB_SERVER || 'local'; - const server2 = process.env.STDB_SERVER || 'http://localhost:3000'; - const modulePath = process.env.STDB_MODULE_PATH || './spacetimedb'; - - // Publish module (creates DB if needed, updates if exists) - await sh('spacetime', [ - 'publish', - '--server', - server, - moduleName, - '--module-path', - modulePath, - ]); - await sh('cargo', [ +// ============================================================================ +// Benchmark +// ============================================================================ + +interface BenchResult { + system: string; + tps: number; +} + +async function runBenchmarkOther(system: string): Promise { + const connectorFactory = (CONNECTORS as any)[system]; + if (!connectorFactory) { + console.log(` ${system}: Unknown connector`); + return null; + } + + const connector = connectorFactory(); + const testMod = await import(`./tests/test-1/${system}.ts`); + const scenario = testMod.default.run; + + const result = await runOne({ + connector, + scenario, + seconds, + concurrency, + accounts, + alpha, + }); + + return { + system, + tps: Math.round(result.tps), + }; +} + +async function runBenchmarkStdb(): Promise { + const moduleName = process.env.STDB_MODULE || 'test-1'; + const server2 = process.env.STDB_SERVER || 'http://localhost:3000'; + + await sh('cargo', [ 'run', //"--quiet", "--manifest-path", "spacetimedb-rust-client/Cargo.toml", "--", - "seed", + "bench", //"--quiet", '--server', server2, "--module", moduleName, - "--accounts", - String(accounts), - "--initial-balance", - String(initialBalance), - ]); - console.log('[spacetimedb] seed complete.'); - } else if (system === 'convex') { - await initConvex(); - } else { - const conn = connector(); - await conn.open(); - await conn.call('seed', { accounts, initialBalance }); - await conn.close(); + "--duration", + `${seconds}s`, + "--connections", + String(concurrency), + "--alpha", + String(alpha), + "--tps-write-path", + "spacetimedb-tps.tmp.log", + ]); + + const tpsStr = fs.readFileSync("spacetimedb-tps.tmp.log", 'utf-8').trim(); + const tps = Number(tpsStr); + if (isNaN(tps)) { + console.warn(`[spacetimedb] Failed to parse TPS from file: ${tpsStr}`); + return null; } - spinner.stop(c('green', '✓ READY')); - } catch (err: any) { - spinner.stop(c('red', `✗ ${err.message}`)); - } -} -// ============================================================================ -// Benchmark -// ============================================================================ - -interface BenchResult { - system: string; - tps: number; + return { + system: "spacetimedb", + tps: Math.round(tps), + }; } -async function runBenchmarkOther(system: string): Promise { - const connectorFactory = (CONNECTORS as any)[system]; - if (!connectorFactory) { - console.log(` ${system}: Unknown connector`); - return null; - } - - const connector = connectorFactory(); - const testMod = await import(`./tests/test-1/${system}.ts`); - const scenario = testMod.default.run; - - const result = await runOne({ - connector, - scenario, - seconds, - concurrency, - accounts, - alpha, - }); - - return { - system, - tps: Math.round(result.tps), - }; -} +async function runBenchmarkPostgres(): Promise { + await sh('go', [ + 'run', + 'main.go', + '--duration', + `${seconds}s`, + "--connections", + String(concurrency), + "--tps-write-path", + "tps", + ], {cwd: 'postgres-go-client'}); + + const tpsStr = fs.readFileSync("postgres-go-client/tps", 'utf-8').trim(); + const tps = Number(tpsStr); + if (isNaN(tps)) { + console.warn(`[postgres] Failed to parse TPS from file: ${tpsStr}`); + return null; + } -async function runBenchmarkStdb(): Promise { - const moduleName = process.env.STDB_MODULE || 'test-1'; - const server2 = process.env.STDB_SERVER || 'http://localhost:3000'; - - await sh('cargo', [ - 'run', - //"--quiet", - "--manifest-path", - "spacetimedb-rust-client/Cargo.toml", - "--", - "bench", - //"--quiet", - '--server', - server2, - "--module", - moduleName, - "--duration", - `${seconds}s`, - "--connections", - String(concurrency), - "--alpha", - String(alpha), - "--tps-write-path", - "spacetimedb-tps.tmp.log", - ]); - - const tpsStr = fs.readFileSync("spacetimedb-tps.tmp.log", 'utf-8').trim(); - const tps = Number(tpsStr); - if (isNaN(tps)) { - console.warn(`[spacetimedb] Failed to parse TPS from file: ${tpsStr}`); - return null; - } - - return { - system: "spacetimedb", - tps: Math.round(tps), - }; + return { + system: "postgres", + tps: Math.round(tps), + }; } async function runBenchmark(system: string): Promise { - if (system === 'spacetimedb') { - return await runBenchmarkStdb(); - } else { - return await runBenchmarkOther(system); - } + if (system === 'spacetimedb') { + return await runBenchmarkStdb(); + } else if (system === 'postgres') { + return await runBenchmarkPostgres(); + } else { + return await runBenchmarkOther(system); + } } // ============================================================================ @@ -367,182 +404,182 @@ async function runBenchmark(system: string): Promise { // ============================================================================ function renderBar(tps: number, maxTps: number, width = 40): string { - const filled = Math.max(1, Math.round((tps / maxTps) * width)); - const bar = '█'.repeat(filled) + '░'.repeat(width - filled); - return c('green', bar); + const filled = Math.max(1, Math.round((tps / maxTps) * width)); + const bar = '█'.repeat(filled) + '░'.repeat(width - filled); + return c('green', bar); } async function displayResults(results: BenchResult[]): Promise { - results.sort((a, b) => b.tps - a.tps); - const maxTps = results[0]?.tps || 1; - - console.log('\n' + c('bold', '═'.repeat(70))); - console.log(c('bold', ' RESULTS')); - console.log(c('bold', '═'.repeat(70)) + '\n'); - - if (noAnimation) { - // Static display - for (const r of results) { - const bar = renderBar(r.tps, maxTps); - const tpsStr = r.tps.toLocaleString().padStart(10); - console.log(` ${r.system.padEnd(14)} ${bar} ${tpsStr} TPS`); + results.sort((a, b) => b.tps - a.tps); + const maxTps = results[0]?.tps || 1; + + console.log('\n' + c('bold', '═'.repeat(70))); + console.log(c('bold', ' RESULTS')); + console.log(c('bold', '═'.repeat(70)) + '\n'); + + if (noAnimation) { + // Static display + for (const r of results) { + const bar = renderBar(r.tps, maxTps); + const tpsStr = r.tps.toLocaleString().padStart(10); + console.log(` ${r.system.padEnd(14)} ${bar} ${tpsStr} TPS`); + } + } else { + // Animated bars growing + const frames = 25; + for (let i = 1; i <= frames; i++) { + const progress = i / frames; + + // Move cursor up to redraw (except first frame) + if (i > 1) { + process.stdout.write(`\x1b[${results.length}A`); + } + + for (const r of results) { + const currentTps = Math.round(r.tps * progress); + const bar = renderBar(currentTps, maxTps); + const tpsStr = currentTps.toLocaleString().padStart(10); + console.log(` ${r.system.padEnd(14)} ${bar} ${tpsStr} TPS`); + } + + await sleep(40); + } } - } else { - // Animated bars growing - const frames = 25; - for (let i = 1; i <= frames; i++) { - const progress = i / frames; - - // Move cursor up to redraw (except first frame) - if (i > 1) { - process.stdout.write(`\x1b[${results.length}A`); - } - - for (const r of results) { - const currentTps = Math.round(r.tps * progress); - const bar = renderBar(currentTps, maxTps); - const tpsStr = currentTps.toLocaleString().padStart(10); - console.log(` ${r.system.padEnd(14)} ${bar} ${tpsStr} TPS`); - } - - await sleep(40); + + // Show comparison + const fastest = results[0]; + const slowest = results[results.length - 1]; + + if (fastest && slowest && fastest.system !== slowest.system && slowest.tps > 0) { + const multiplier = Math.round(fastest.tps / slowest.tps); + + console.log(''); + + if (!noAnimation) { + // Animated reveal of the comparison box + await sleep(200); + } + + const boxWidth = 60; + const msgText = `${fastest.system} is ${multiplier}x FASTER than ${slowest.system}!`; + const msgWithEmoji = `🚀 ${msgText} 🚀`; + // Emojis are 2 display columns each, so total display width = text + 4 (2 emojis) + 2 (spaces) + const displayWidth = msgText.length + 6; + const msgPadding = Math.floor((boxWidth - displayWidth) / 2); + const rightPadding = boxWidth - msgPadding - displayWidth; + + console.log(' ' + c('cyan', '╔' + '═'.repeat(boxWidth) + '╗')); + console.log(' ' + c('cyan', '║') + ' '.repeat(boxWidth) + c('cyan', '║')); + console.log( + ' ' + + c('cyan', '║') + + ' '.repeat(msgPadding) + + c('bold', c('green', msgWithEmoji)) + + ' '.repeat(rightPadding) + + c('cyan', '║'), + ); + console.log(' ' + c('cyan', '║') + ' '.repeat(boxWidth) + c('cyan', '║')); + console.log(' ' + c('cyan', '╚' + '═'.repeat(boxWidth) + '╝')); } - } +} - // Show comparison - const fastest = results[0]; - const slowest = results[results.length - 1]; +// ============================================================================ +// Main +// ============================================================================ - if (fastest && slowest && fastest.system !== slowest.system && slowest.tps > 0) { - const multiplier = Math.round(fastest.tps / slowest.tps); +async function main() { + const headerWidth = 59; + const headerText = 'SpacetimeDB Benchmark Demo'; + const headerPadding = Math.floor((headerWidth - headerText.length) / 2); + const headerPaddedText = + ' '.repeat(headerPadding) + + headerText + + ' '.repeat(headerWidth - headerPadding - headerText.length); console.log(''); + console.log(c('bold', c('cyan', ' ╔' + '═'.repeat(headerWidth) + '╗'))); + console.log(c('bold', c('cyan', ' ║') + headerPaddedText + c('cyan', '║'))); + console.log(c('bold', c('cyan', ' ╚' + '═'.repeat(headerWidth) + '╝'))); + console.log(''); - if (!noAnimation) { - // Animated reveal of the comparison box - await sleep(200); - } - - const boxWidth = 60; - const msgText = `${fastest.system} is ${multiplier}x FASTER than ${slowest.system}!`; - const msgWithEmoji = `🚀 ${msgText} 🚀`; - // Emojis are 2 display columns each, so total display width = text + 4 (2 emojis) + 2 (spaces) - const displayWidth = msgText.length + 6; - const msgPadding = Math.floor((boxWidth - displayWidth) / 2); - const rightPadding = boxWidth - msgPadding - displayWidth; - - console.log(' ' + c('cyan', '╔' + '═'.repeat(boxWidth) + '╗')); - console.log(' ' + c('cyan', '║') + ' '.repeat(boxWidth) + c('cyan', '║')); console.log( - ' ' + - c('cyan', '║') + - ' '.repeat(msgPadding) + - c('bold', c('green', msgWithEmoji)) + - ' '.repeat(rightPadding) + - c('cyan', '║'), + ` ${c('dim', 'Config:')} ${seconds}s, ${concurrency} connections, alpha=${alpha}`, ); - console.log(' ' + c('cyan', '║') + ' '.repeat(boxWidth) + c('cyan', '║')); - console.log(' ' + c('cyan', '╚' + '═'.repeat(boxWidth) + '╝')); - } -} + console.log(` ${c('dim', 'Systems:')} ${systems.join(', ')}\n`); -// ============================================================================ -// Main -// ============================================================================ + // Step 1: Check services + console.log(c('bold', ' [1/4] Checking services...\n')); -async function main() { - const headerWidth = 59; - const headerText = 'SpacetimeDB Benchmark Demo'; - const headerPadding = Math.floor((headerWidth - headerText.length) / 2); - const headerPaddedText = - ' '.repeat(headerPadding) + - headerText + - ' '.repeat(headerWidth - headerPadding - headerText.length); - - console.log(''); - console.log(c('bold', c('cyan', ' ╔' + '═'.repeat(headerWidth) + '╗'))); - console.log(c('bold', c('cyan', ' ║') + headerPaddedText + c('cyan', '║'))); - console.log(c('bold', c('cyan', ' ╚' + '═'.repeat(headerWidth) + '╝'))); - console.log(''); - - console.log( - ` ${c('dim', 'Config:')} ${seconds}s, ${concurrency} connections, alpha=${alpha}`, - ); - console.log(` ${c('dim', 'Systems:')} ${systems.join(', ')}\n`); - - // Step 1: Check services - console.log(c('bold', ' [1/4] Checking services...\n')); - - for (const system of systems) { - const ok = await checkService(system); - if (!ok) { - console.log( - `\n${c('red', ' ERROR:')} ${system} is not running. Exiting.`, - ); - process.exit(1); + for (const system of systems) { + const ok = await checkService(system); + if (!ok) { + console.log( + `\n${c('red', ' ERROR:')} ${system} is not running. Exiting.`, + ); + process.exit(1); + } + } + + // Step 2: Prep/seed + if (!skipPrep) { + console.log('\n' + c('bold', ' [2/4] Preparing databases...\n')); + for (const system of systems) { + await prepSystem(system); + } + } else { + console.log( + '\n' + + c('bold', ' [2/4] Preparing databases...') + + c('dim', ' (skipped)\n'), + ); } - } - // Step 2: Prep/seed - if (!skipPrep) { - console.log('\n' + c('bold', ' [2/4] Preparing databases...\n')); + // Step 3: Run benchmarks + console.log('\n' + c('bold', ' [3/4] Running benchmarks...\n')); + + const results: BenchResult[] = []; for (const system of systems) { - await prepSystem(system); + const spinner = createSpinner(`${system.padEnd(12)} benchmarking`); + const result = await runBenchmark(system); + if (result && result.tps > 0) { + spinner.stop(c('green', `✓ ${result.tps.toLocaleString()} TPS`)); + results.push(result); + } else { + spinner.stop(c('red', `✗ FAILED (0 completed transactions)`)); + } } - } else { - console.log( - '\n' + - c('bold', ' [2/4] Preparing databases...') + - c('dim', ' (skipped)\n'), - ); - } - - // Step 3: Run benchmarks - console.log('\n' + c('bold', ' [3/4] Running benchmarks...\n')); - - const results: BenchResult[] = []; - for (const system of systems) { - const spinner = createSpinner(`${system.padEnd(12)} benchmarking`); - const result = await runBenchmark(system); - if (result && result.tps > 0) { - spinner.stop(c('green', `✓ ${result.tps.toLocaleString()} TPS`)); - results.push(result); - } else { - spinner.stop(c('red', `✗ FAILED (0 completed transactions)`)); + + // Step 4: Display results + if (results.length > 0) { + await displayResults(results); + + // Save to JSON + const runsDir = join(process.cwd(), 'runs'); + await mkdir(runsDir, {recursive: true}); + const outFile = join( + runsDir, + `demo-${new Date().toISOString().replace(/[:.]/g, '-')}.json`, + ); + await writeFile( + outFile, + JSON.stringify( + { + timestamp: new Date().toISOString(), + config: {seconds, concurrency, alpha, accounts}, + results: results.map((r) => ({ + system: r.system, + tps: r.tps, + })), + }, + null, + 2, + ), + ); + console.log(`\n${c('dim', ` Results saved to: ${outFile}`)}\n`); } - } - - // Step 4: Display results - if (results.length > 0) { - await displayResults(results); - - // Save to JSON - const runsDir = join(process.cwd(), 'runs'); - await mkdir(runsDir, { recursive: true }); - const outFile = join( - runsDir, - `demo-${new Date().toISOString().replace(/[:.]/g, '-')}.json`, - ); - await writeFile( - outFile, - JSON.stringify( - { - timestamp: new Date().toISOString(), - config: { seconds, concurrency, alpha, accounts }, - results: results.map((r) => ({ - system: r.system, - tps: r.tps, - })), - }, - null, - 2, - ), - ); - console.log(`\n${c('dim', ` Results saved to: ${outFile}`)}\n`); - } } main().catch((err) => { - console.error('\n' + c('red', ' ERROR:'), err.message); - process.exit(1); + console.error('\n' + c('red', ' ERROR:'), err.message); + process.exit(1); }); diff --git a/templates/keynote-2/src/scenario_recipes/postgres_no_rpc_single_call.ts b/templates/keynote-2/src/scenario_recipes/postgres_no_rpc_single_call.ts new file mode 100644 index 00000000000..0ef3ca43e93 --- /dev/null +++ b/templates/keynote-2/src/scenario_recipes/postgres_no_rpc_single_call.ts @@ -0,0 +1,18 @@ +import type { RpcConnector } from '../core/connectors'; + +export async function postgres_no_rpc_single_call( + conn: unknown, + from: number, + to: number, + amount: number, +): Promise { + if (from === to || amount <= 0) return; + const connector = conn as RpcConnector; + const fn = 'transfer'; + try { + await connector.call(fn, {from: from, to: to, amount: amount}); + } catch (err) { + console.error(`[postgres_single_call] ${fn} failed:`, err); + } + return; +} diff --git a/templates/keynote-2/src/tests/test-1/postgres_no_rpc.ts b/templates/keynote-2/src/tests/test-1/postgres_no_rpc.ts new file mode 100644 index 00000000000..8ed011ffd55 --- /dev/null +++ b/templates/keynote-2/src/tests/test-1/postgres_no_rpc.ts @@ -0,0 +1,7 @@ +import { postgres_no_rpc_single_call } from '../../scenario_recipes/postgres_no_rpc_single_call.ts'; + +export default { + system: 'postgres_no_rpc', + label: 'postgres_no_rpc:single_call', + run: postgres_no_rpc_single_call, +};