diff --git a/Cargo.lock b/Cargo.lock index 79823fbcf32..2a78a7f8670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,7 +449,7 @@ name = "benchmarks-module" version = "0.1.0" dependencies = [ "anyhow", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -1103,7 +1103,7 @@ dependencies = [ [[package]] name = "connect_disconnect_client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-sdk", @@ -3545,7 +3545,7 @@ name = "keynote-benchmarks" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -3931,7 +3931,7 @@ version = "0.0.0" dependencies = [ "anyhow", "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -5099,7 +5099,7 @@ name = "perf-test-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -5507,12 +5507,12 @@ dependencies = [ [[package]] name = "procedure-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", "serde_json", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-sdk", "test-counter", ] @@ -5716,7 +5716,7 @@ name = "quickstart-chat-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6813,7 +6813,7 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6823,7 +6823,7 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -6833,12 +6833,12 @@ dependencies = [ "anyhow", "log", "paste", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] name = "sdk-unreal-test-harness" -version = "1.11.3" +version = "1.12.0" dependencies = [ "serial_test", "spacetimedb-testing", @@ -7284,7 +7284,7 @@ name = "spacetime-module" version = "0.1.0" dependencies = [ "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -7307,7 +7307,7 @@ dependencies = [ [[package]] name = "spacetimedb" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytemuck", @@ -7320,29 +7320,29 @@ dependencies = [ "rand 0.8.5", "scoped-tls", "serde_json", - "spacetimedb-bindings-macro 1.11.3", - "spacetimedb-bindings-sys 1.11.3", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-bindings-macro 1.12.0", + "spacetimedb-bindings-sys 1.12.0", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-query-builder", "trybuild", ] [[package]] name = "spacetimedb-auth" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "serde", "serde_json", "serde_with", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", ] [[package]] name = "spacetimedb-bench" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "anyhow", @@ -7372,11 +7372,11 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-datastore", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-standalone", "spacetimedb-table", @@ -7405,13 +7405,13 @@ dependencies = [ [[package]] name = "spacetimedb-bindings-macro" -version = "1.11.3" +version = "1.12.0" dependencies = [ "heck 0.4.1", "humantime", "proc-macro2", "quote", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "syn 2.0.107", ] @@ -7426,14 +7426,14 @@ dependencies = [ [[package]] name = "spacetimedb-bindings-sys" -version = "1.11.3" +version = "1.12.0" dependencies = [ - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", ] [[package]] name = "spacetimedb-cli" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "assert_cmd", @@ -7481,9 +7481,9 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-guard", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "syntect", "tabled", @@ -7508,7 +7508,7 @@ dependencies = [ [[package]] name = "spacetimedb-client-api" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-stream", @@ -7547,7 +7547,7 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-datastore", "spacetimedb-jsonwebtoken", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-schema", "tempfile", @@ -7565,7 +7565,7 @@ dependencies = [ [[package]] name = "spacetimedb-client-api-messages" -version = "1.11.3" +version = "1.12.0" dependencies = [ "bytes", "bytestring", @@ -7579,16 +7579,16 @@ dependencies = [ "serde_json", "serde_with", "smallvec", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "strum", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-codegen" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -7597,15 +7597,15 @@ dependencies = [ "itertools 0.12.1", "regex", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "spacetimedb-testing", ] [[package]] name = "spacetimedb-commitlog" -version = "1.11.3" +version = "1.12.0" dependencies = [ "async-stream", "bitflags 2.10.0", @@ -7626,8 +7626,8 @@ dependencies = [ "spacetimedb-commitlog", "spacetimedb-fs-utils", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "tempfile", "thiserror 1.0.69", "tokio", @@ -7638,7 +7638,7 @@ dependencies = [ [[package]] name = "spacetimedb-core" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "arrayvec", @@ -7721,14 +7721,14 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-jsonwebtoken", "spacetimedb-jwks", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", "spacetimedb-paths", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-snapshot", "spacetimedb-subscription", @@ -7764,7 +7764,7 @@ dependencies = [ [[package]] name = "spacetimedb-data-structures" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "crossbeam-queue", @@ -7778,7 +7778,7 @@ dependencies = [ [[package]] name = "spacetimedb-datastore" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -7798,11 +7798,11 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-durability", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-metrics", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-snapshot", "spacetimedb-table", @@ -7813,7 +7813,7 @@ dependencies = [ [[package]] name = "spacetimedb-durability" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "futures", @@ -7823,7 +7823,7 @@ dependencies = [ "spacetimedb-commitlog", "spacetimedb-fs-utils", "spacetimedb-paths", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "tempfile", "thiserror 1.0.69", "tokio", @@ -7832,22 +7832,22 @@ dependencies = [ [[package]] name = "spacetimedb-execution" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "itertools 0.12.1", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-sql-parser", "spacetimedb-table", ] [[package]] name = "spacetimedb-expr" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bigdecimal", @@ -7855,10 +7855,10 @@ dependencies = [ "derive_more 0.99.20", "ethnum", "pretty_assertions", - "spacetimedb 1.11.3", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb 1.12.0", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-sql-parser", "thiserror 1.0.69", @@ -7866,7 +7866,7 @@ dependencies = [ [[package]] name = "spacetimedb-fs-utils" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "fs2", @@ -7880,7 +7880,7 @@ dependencies = [ [[package]] name = "spacetimedb-guard" -version = "1.11.3" +version = "1.12.0" dependencies = [ "portpicker", "reqwest 0.12.24", @@ -7938,7 +7938,7 @@ dependencies = [ [[package]] name = "spacetimedb-lib" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bitflags 2.10.0", @@ -7957,17 +7957,17 @@ dependencies = [ "ron", "serde", "serde_json", - "spacetimedb-bindings-macro 1.11.3", + "spacetimedb-bindings-macro 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-memory-usage" -version = "1.11.3" +version = "1.12.0" dependencies = [ "decorum", "ethnum", @@ -7977,7 +7977,7 @@ dependencies = [ [[package]] name = "spacetimedb-metrics" -version = "1.11.3" +version = "1.12.0" dependencies = [ "arrayvec", "itertools 0.12.1", @@ -7987,7 +7987,7 @@ dependencies = [ [[package]] name = "spacetimedb-paths" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "chrono", @@ -8003,7 +8003,7 @@ dependencies = [ [[package]] name = "spacetimedb-pg" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8014,22 +8014,22 @@ dependencies = [ "pgwire", "spacetimedb-client-api", "spacetimedb-client-api-messages", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "thiserror 1.0.69", "tokio", ] [[package]] name = "spacetimedb-physical-plan" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "derive_more 0.99.20", "either", "pretty_assertions", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", "spacetimedb-schema", "spacetimedb-sql-parser", "spacetimedb-table", @@ -8049,7 +8049,7 @@ dependencies = [ [[package]] name = "spacetimedb-primitives" -version = "1.11.3" +version = "1.12.0" dependencies = [ "bitflags 2.10.0", "either", @@ -8062,7 +8062,7 @@ dependencies = [ [[package]] name = "spacetimedb-query" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "itertools 0.12.1", @@ -8070,18 +8070,18 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-execution", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-sql-parser", "spacetimedb-table", ] [[package]] name = "spacetimedb-query-builder" -version = "1.11.3" +version = "1.12.0" dependencies = [ - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", ] [[package]] @@ -8112,7 +8112,7 @@ dependencies = [ [[package]] name = "spacetimedb-sats" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "anyhow", @@ -8137,17 +8137,17 @@ dependencies = [ "serde_json", "sha3", "smallvec", - "spacetimedb-bindings-macro 1.11.3", + "spacetimedb-bindings-macro 1.12.0", "spacetimedb-memory-usage", "spacetimedb-metrics", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "thiserror 1.0.69", "uuid", ] [[package]] name = "spacetimedb-schema" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "derive_more 0.99.20", @@ -8163,10 +8163,10 @@ dependencies = [ "serial_test", "smallvec", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-sql-parser", "spacetimedb-testing", "termcolor", @@ -8177,7 +8177,7 @@ dependencies = [ [[package]] name = "spacetimedb-sdk" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anymap", "base64 0.21.7", @@ -8196,10 +8196,10 @@ dependencies = [ "rand 0.9.2", "spacetimedb-client-api-messages", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-metrics", "spacetimedb-query-builder", - "spacetimedb-sats 1.11.3", + "spacetimedb-sats 1.12.0", "spacetimedb-testing", "thiserror 1.0.69", "tokio", @@ -8208,7 +8208,7 @@ dependencies = [ [[package]] name = "spacetimedb-snapshot" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "blake3", @@ -8225,10 +8225,10 @@ dependencies = [ "spacetimedb-datastore", "spacetimedb-durability", "spacetimedb-fs-utils", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-table", "tempfile", @@ -8241,17 +8241,17 @@ dependencies = [ [[package]] name = "spacetimedb-sql-parser" -version = "1.11.3" +version = "1.12.0" dependencies = [ "derive_more 0.99.20", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "sqlparser", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-standalone" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8276,7 +8276,7 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-datastore", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-pg", "spacetimedb-schema", @@ -8293,20 +8293,20 @@ dependencies = [ [[package]] name = "spacetimedb-subscription" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-execution", "spacetimedb-expr", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-physical-plan", - "spacetimedb-primitives 1.11.3", + "spacetimedb-primitives 1.12.0", "spacetimedb-query", ] [[package]] name = "spacetimedb-table" -version = "1.11.3" +version = "1.12.0" dependencies = [ "ahash 0.8.12", "blake3", @@ -8324,17 +8324,17 @@ dependencies = [ "rand 0.9.2", "smallvec", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-memory-usage", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "thiserror 1.0.69", ] [[package]] name = "spacetimedb-testing" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -8352,7 +8352,7 @@ dependencies = [ "spacetimedb-client-api", "spacetimedb-core", "spacetimedb-data-structures", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-paths", "spacetimedb-schema", "spacetimedb-standalone", @@ -8363,7 +8363,7 @@ dependencies = [ [[package]] name = "spacetimedb-update" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "bytes", @@ -8388,7 +8388,7 @@ dependencies = [ [[package]] name = "spacetimedb-vm" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "arrayvec", @@ -8398,9 +8398,9 @@ dependencies = [ "smallvec", "spacetimedb-data-structures", "spacetimedb-execution", - "spacetimedb-lib 1.11.3", - "spacetimedb-primitives 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-primitives 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-schema", "spacetimedb-table", "tempfile", @@ -8476,7 +8476,7 @@ dependencies = [ [[package]] name = "sqltest" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "async-trait", @@ -8495,8 +8495,8 @@ dependencies = [ "rust_decimal", "spacetimedb-core", "spacetimedb-datastore", - "spacetimedb-lib 1.11.3", - "spacetimedb-sats 1.11.3", + "spacetimedb-lib 1.12.0", + "spacetimedb-sats 1.12.0", "spacetimedb-vm", "sqllogictest", "sqllogictest-engines", @@ -8891,7 +8891,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", @@ -8903,7 +8903,7 @@ dependencies = [ [[package]] name = "test-counter" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "spacetimedb-data-structures", @@ -9633,7 +9633,7 @@ version = "0.1.0" dependencies = [ "anyhow", "log", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", ] [[package]] @@ -9835,11 +9835,11 @@ dependencies = [ [[package]] name = "view-client" -version = "1.11.3" +version = "1.12.0" dependencies = [ "anyhow", "env_logger 0.10.2", - "spacetimedb-lib 1.11.3", + "spacetimedb-lib 1.12.0", "spacetimedb-sdk", "test-counter", ] @@ -11013,7 +11013,7 @@ dependencies = [ "reqwest 0.12.24", "serde", "serde_json", - "spacetimedb 1.11.3", + "spacetimedb 1.12.0", "spacetimedb-guard", "tempfile", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index 3fd9392aca3..9b2a4b29070 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,46 +100,46 @@ inherits = "release" debug = true [workspace.package] -version = "1.11.3" +version = "1.12.0" edition = "2021" # update rust-toolchain.toml too! rust-version = "1.90.0" [workspace.dependencies] -spacetimedb = { path = "crates/bindings", version = "=1.11.3" } -spacetimedb-auth = { path = "crates/auth", version = "=1.11.3" } -spacetimedb-bindings-macro = { path = "crates/bindings-macro", version = "=1.11.3" } -spacetimedb-bindings-sys = { path = "crates/bindings-sys", version = "=1.11.3" } -spacetimedb-cli = { path = "crates/cli", version = "=1.11.3" } -spacetimedb-client-api = { path = "crates/client-api", version = "=1.11.3" } -spacetimedb-client-api-messages = { path = "crates/client-api-messages", version = "=1.11.3" } -spacetimedb-codegen = { path = "crates/codegen", version = "=1.11.3" } -spacetimedb-commitlog = { path = "crates/commitlog", version = "=1.11.3" } -spacetimedb-core = { path = "crates/core", version = "=1.11.3" } -spacetimedb-data-structures = { path = "crates/data-structures", version = "=1.11.3" } -spacetimedb-datastore = { path = "crates/datastore", version = "=1.11.3" } -spacetimedb-durability = { path = "crates/durability", version = "=1.11.3" } -spacetimedb-execution = { path = "crates/execution", version = "=1.11.3" } -spacetimedb-expr = { path = "crates/expr", version = "=1.11.3" } -spacetimedb-guard = { path = "crates/guard", version = "=1.11.3" } -spacetimedb-lib = { path = "crates/lib", default-features = false, version = "=1.11.3" } -spacetimedb-memory-usage = { path = "crates/memory-usage", version = "=1.11.3", default-features = false } -spacetimedb-metrics = { path = "crates/metrics", version = "=1.11.3" } -spacetimedb-paths = { path = "crates/paths", version = "=1.11.3" } -spacetimedb-pg = { path = "crates/pg", version = "=1.11.3" } -spacetimedb-physical-plan = { path = "crates/physical-plan", version = "=1.11.3" } -spacetimedb-primitives = { path = "crates/primitives", version = "=1.11.3" } -spacetimedb-query = { path = "crates/query", version = "=1.11.3" } -spacetimedb-sats = { path = "crates/sats", version = "=1.11.3" } -spacetimedb-schema = { path = "crates/schema", version = "=1.11.3" } -spacetimedb-standalone = { path = "crates/standalone", version = "=1.11.3" } -spacetimedb-sql-parser = { path = "crates/sql-parser", version = "=1.11.3" } -spacetimedb-table = { path = "crates/table", version = "=1.11.3" } -spacetimedb-vm = { path = "crates/vm", version = "=1.11.3" } -spacetimedb-fs-utils = { path = "crates/fs-utils", version = "=1.11.3" } -spacetimedb-snapshot = { path = "crates/snapshot", version = "=1.11.3" } -spacetimedb-subscription = { path = "crates/subscription", version = "=1.11.3" } -spacetimedb-query-builder = { path = "crates/query-builder", version = "=1.11.3" } +spacetimedb = { path = "crates/bindings", version = "=1.12.0" } +spacetimedb-auth = { path = "crates/auth", version = "=1.12.0" } +spacetimedb-bindings-macro = { path = "crates/bindings-macro", version = "=1.12.0" } +spacetimedb-bindings-sys = { path = "crates/bindings-sys", version = "=1.12.0" } +spacetimedb-cli = { path = "crates/cli", version = "=1.12.0" } +spacetimedb-client-api = { path = "crates/client-api", version = "=1.12.0" } +spacetimedb-client-api-messages = { path = "crates/client-api-messages", version = "=1.12.0" } +spacetimedb-codegen = { path = "crates/codegen", version = "=1.12.0" } +spacetimedb-commitlog = { path = "crates/commitlog", version = "=1.12.0" } +spacetimedb-core = { path = "crates/core", version = "=1.12.0" } +spacetimedb-data-structures = { path = "crates/data-structures", version = "=1.12.0" } +spacetimedb-datastore = { path = "crates/datastore", version = "=1.12.0" } +spacetimedb-durability = { path = "crates/durability", version = "=1.12.0" } +spacetimedb-execution = { path = "crates/execution", version = "=1.12.0" } +spacetimedb-expr = { path = "crates/expr", version = "=1.12.0" } +spacetimedb-guard = { path = "crates/guard", version = "=1.12.0" } +spacetimedb-lib = { path = "crates/lib", default-features = false, version = "=1.12.0" } +spacetimedb-memory-usage = { path = "crates/memory-usage", version = "=1.12.0", default-features = false } +spacetimedb-metrics = { path = "crates/metrics", version = "=1.12.0" } +spacetimedb-paths = { path = "crates/paths", version = "=1.12.0" } +spacetimedb-pg = { path = "crates/pg", version = "=1.12.0" } +spacetimedb-physical-plan = { path = "crates/physical-plan", version = "=1.12.0" } +spacetimedb-primitives = { path = "crates/primitives", version = "=1.12.0" } +spacetimedb-query = { path = "crates/query", version = "=1.12.0" } +spacetimedb-sats = { path = "crates/sats", version = "=1.12.0" } +spacetimedb-schema = { path = "crates/schema", version = "=1.12.0" } +spacetimedb-standalone = { path = "crates/standalone", version = "=1.12.0" } +spacetimedb-sql-parser = { path = "crates/sql-parser", version = "=1.12.0" } +spacetimedb-table = { path = "crates/table", version = "=1.12.0" } +spacetimedb-vm = { path = "crates/vm", version = "=1.12.0" } +spacetimedb-fs-utils = { path = "crates/fs-utils", version = "=1.12.0" } +spacetimedb-snapshot = { path = "crates/snapshot", version = "=1.12.0" } +spacetimedb-subscription = { path = "crates/subscription", version = "=1.12.0" } +spacetimedb-query-builder = { path = "crates/query-builder", version = "=1.12.0" } # Prevent `ahash` from pulling in `getrandom` by disabling default features. # Modules use `getrandom02` and we need to prevent an incompatible version diff --git a/LICENSE.txt b/LICENSE.txt index 3e4177f2240..105d78746b8 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -5,7 +5,7 @@ Business Source License 1.1 Parameters Licensor: Clockwork Laboratories, Inc. -Licensed Work: SpacetimeDB 1.11.3 +Licensed Work: SpacetimeDB 1.12.0 The Licensed Work is (c) 2023 Clockwork Laboratories, Inc. @@ -21,7 +21,7 @@ Additional Use Grant: You may make use of the Licensed Work provided your Licensed Work by creating tables whose schemas are controlled by such third parties. -Change Date: 2031-01-14 +Change Date: 2031-01-24 Change License: GNU Affero General Public License v3.0 with a linking exception diff --git a/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj b/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj index 01036e7a28f..48df32eb1dd 100644 --- a/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj +++ b/crates/bindings-csharp/BSATN.Codegen/BSATN.Codegen.csproj @@ -2,7 +2,7 @@ SpacetimeDB.BSATN.Codegen - 1.11.2 + 1.12.0 SpacetimeDB BSATN Codegen The SpacetimeDB BSATN Codegen implements the Roslyn incremental generators for BSATN serialization/deserialization in C#. diff --git a/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj b/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj index 1bd693c0d9d..38c8c9b1ea3 100644 --- a/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj +++ b/crates/bindings-csharp/BSATN.Runtime/BSATN.Runtime.csproj @@ -1,7 +1,7 @@ SpacetimeDB.BSATN.Runtime - 1.11.2 + 1.12.0 SpacetimeDB BSATN Runtime The SpacetimeDB BSATN Runtime implements APIs for BSATN serialization/deserialization in C#. true diff --git a/crates/bindings-csharp/Codegen/Codegen.csproj b/crates/bindings-csharp/Codegen/Codegen.csproj index 3618428a00f..a38a9be1b32 100644 --- a/crates/bindings-csharp/Codegen/Codegen.csproj +++ b/crates/bindings-csharp/Codegen/Codegen.csproj @@ -1,7 +1,7 @@ SpacetimeDB.Codegen - 1.11.2 + 1.12.0 SpacetimeDB Module Codegen The SpacetimeDB Codegen implements the Roslyn incremental generators for writing SpacetimeDB modules in C#. diff --git a/crates/bindings-csharp/Runtime/Runtime.csproj b/crates/bindings-csharp/Runtime/Runtime.csproj index 2d4329c965e..268cffcf592 100644 --- a/crates/bindings-csharp/Runtime/Runtime.csproj +++ b/crates/bindings-csharp/Runtime/Runtime.csproj @@ -2,7 +2,7 @@ SpacetimeDB.Runtime - 1.11.2 + 1.12.0 SpacetimeDB Module Runtime The SpacetimeDB Runtime implements the database runtime bindings for writing SpacetimeDB modules in C#. diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index e2b50cb78dc..d89b6773042 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -1,6 +1,6 @@ { "name": "spacetimedb", - "version": "1.11.2", + "version": "1.12.0", "description": "API and ABI bindings for the SpacetimeDB TypeScript module library", "homepage": "https://github.com/clockworklabs/SpacetimeDB#readme", "bugs": { diff --git a/crates/cli/src/subcommands/project/typescript/package._json b/crates/cli/src/subcommands/project/typescript/package._json index 969a85f8dce..40c51e78b41 100644 --- a/crates/cli/src/subcommands/project/typescript/package._json +++ b/crates/cli/src/subcommands/project/typescript/package._json @@ -10,6 +10,6 @@ "author": "", "license": "ISC", "dependencies": { - "spacetimedb": "1.11.*" + "spacetimedb": "1.12.*" } } \ No newline at end of file diff --git a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap index c4fbcde807f..c2dee2a19b0 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap @@ -547,7 +547,7 @@ const proceduresSchema = __procedures( /** The remote SpacetimeDB module schema, both runtime and type information. */ const REMOTE_MODULE = { versionInfo: { - cliVersion: "1.11.3" as const, + cliVersion: "1.12.0" as const, }, tables: tablesSchema.schemaType.tables, reducers: reducersSchema.reducersType.reducers, diff --git a/demo/Blackholio/server-csharp/StdbModule.csproj b/demo/Blackholio/server-csharp/StdbModule.csproj index 9610e3caa40..45e503f262a 100644 --- a/demo/Blackholio/server-csharp/StdbModule.csproj +++ b/demo/Blackholio/server-csharp/StdbModule.csproj @@ -13,7 +13,7 @@ - + diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 830c04f6c43..e9fac33c1d4 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,26 +4,23 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 34 +- **Total failures analyzed**: 35 --- # Analysis of SpacetimeDB Benchmark Failures -This analysis focuses on test failures within SpacetimeDB benchmarks, specifically categorized by language and mode, providing actionable insights for documentation improvements to reduce these errors. +## Rust / rustdoc_json Failures (9 total) ---- - -## Rust / rustdoc_json Failures (7 total) +### Compile/Publish Errors -### Compile/Publish Errors (3 failures) +#### t_002_scheduled_table & t_017_scheduled_columns -#### 1. t_002_scheduled_table -- **The generated code**: +1. **The generated code**: ```rust - use spacetimedb::{ReducerContext, ScheduleAt, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -31,21 +28,22 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical scheduled_at: ScheduleAt, } - #[spacetimedb::reducer(init)] - pub fn init(ctx: &ReducerContext) { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), - }); + #[reducer(scheduled)] + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { } - #[spacetimedb::reducer] - pub fn tick(_ctx: &ReducerContext) { - log::info!("tick"); + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat_micros(50_000), + }); + } } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -72,353 +70,245 @@ This analysis focuses on test failures within SpacetimeDB benchmarks, specifical } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The generated code incorrectly used `ScheduleAt::repeat_micros(50_000)` instead of `ScheduleAt::Interval(Duration::from_millis(50).into())`. - - The reducer function lacked parameters that were required. +3. **The error**: + ```plaintext + publish_error: spacetime publish failed (exit=1) + ``` + +4. **Explain the difference**: + - The LLM used incorrect syntax for reducer attributes and data type for `scheduled_at`. The expected code uses `scheduled` as an attribute and specifies data converts. -- **Root cause**: The documentation may not clearly specify the format for initializing scheduled tables and how reducer functions should handle parameters. +5. **Root cause**: + - The documentation lacks clarity on the correct use of the `scheduled` attribute and how to handle the `ScheduleAt` type properly. -- **Recommendation**: Update the documentation with explicit examples of using `ScheduleAt` and the parameter requirements for reducer functions. +6. **Recommendation**: + - Update documentation to clearly explain the usage and requirement of reducer attributes and how to define the `ScheduleAt` type. Show examples covering both attribute definitions and data types. --- -#### 2. t_007_crud -- **The generated code**: +#### t_019_many_to_many + +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext}; + use spacetimedb::{table, reducer, ReducerContext, Table}; #[table(name = user)] pub struct User { #[primary_key] - id: i32, + user_id: i32, name: String, - age: i32, - active: bool, } - #[reducer] - pub fn crud(ctx: &ReducerContext) { - let users = ctx.db.user(); - - users.insert(User { - id: 1, - name: "Alice".to_string(), - age: 30, - active: true, - }); - - users.insert(User { - id: 2, - name: "Bob".to_string(), - age: 22, - active: false, - }); - - if let Some(mut u) = users.id().find(1) { - u.name = "Alice2".to_string(); - u.age = 31; - u.active = false; - users.id().update(u); - } + #[table(name = group)] + pub struct Group { + #[primary_key] + group_id: i32, + title: String, + } - users.id().delete(&2); + #[table( + name = membership, + index(name = by_user, btree(columns = [user_id])), + index(name = by_group, btree(columns = [group_id])) + )] + pub struct Membership { + #[primary_key] + id: i32, + user_id: i32, + group_id: i32, } ``` - -- **The golden example**: + +2. **The golden example**: ```rust use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = user)] pub struct User { #[primary_key] - pub id: i32, + pub user_id: i32, pub name: String, - pub age: i32, - pub active: bool, } - #[reducer] - pub fn crud(ctx: &ReducerContext) { - ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); - ctx.db.user().insert(User { id: 2, name: "Bob".into(), age: 22, active: false }); - ctx.db.user().id().update(User { id: 1, name: "Alice2".into(), age: 31, active: false }); - ctx.db.user().id().delete(2); + #[table(name = group)] + pub struct Group { + #[primary_key] + pub group_id: i32, + pub title: String, + } + + #[table( + name = membership, + index(name = by_user, btree(columns = [user_id])), + index(name = by_group, btree(columns = [group_id])) + )] + pub struct Membership { + #[primary_key] + pub id: i32, + pub user_id: i32, + pub group_id: i32, } ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` +3. **The error**: + ```plaintext + publish_error: spacetime publish failed (exit=1) + ``` -- **Explain the difference**: - - The generated code did not declare the fields of the `User` struct as public, which is a requirement for fields of structs used with SpacetimeDB. - - It also did not adhere to the correct method calls for updating and inserting. +4. **Explain the difference**: + - The generated code does not mark primary key fields as public, whereas the expected code does, which can cause access issues. -- **Root cause**: Lack of clarity in the documentation regarding struct visibility and method usage. +5. **Root cause**: + - Lack of clarity in the documentation regarding the access level of struct fields in SpacetimeDB. -- **Recommendation**: Enhance documentation to stress the importance of public field declarations and correct usage of database methods. +6. **Recommendation**: + - Add notes in the documentation stating that public access is required for fields in database schemas. --- -#### 3. t_017_scheduled_columns -- **The generated code**: +### Other Failures + +#### t_003_struct_in_table + +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; + use spacetimedb::ReducerContext; + use spacetimedb::SpacetimeType; - #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, + #[derive(SpacetimeType)] + pub struct Position { + pub x: i32, + pub y: i32, } - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(50_000), - }); - } + #[spacetimedb::table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, } - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { - } + #[spacetimedb::reducer(init)] + pub fn init(_ctx: &ReducerContext) {} ``` - -- **The golden example**: + +2. **The golden example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; + use spacetimedb::{table, SpacetimeType}; - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, + #[derive(SpacetimeType, Clone, Debug)] + pub struct Position { + pub x: i32, + pub y: i32, } - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { + #[table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, } + ``` - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } +3. **The error**: + ```plaintext + schema_parity: reducers differ - expected [], got ["init()"] ``` -- **The error**: `publish_error: spacetime publish failed (exit=1)` +4. **Explain the difference**: + - The generated function `init` is not necessary, and it's leading to additional unwanted complexity that results in a failure. -- **Explain the difference**: - - The LLM incorrectly specified the scheduled column and did not update the initialization logic for accurate type conversion. - -- **Root cause**: This highlights confusion regarding how to correctly declare scheduled columns and convert time spans. +5. **Root cause**: + - The documentation does not specify when reducers are required, which should be clarified. -- **Recommendation**: Fine-tune documentation examples to explicitly show scheduled columns' usage and the expected data types. +6. **Recommendation**: + - Clarify in the docs that reducers are only needed when initializing data or other specific purposes. --- -### Other Failures (4 failures) +#### t_013_spacetime_sum_type -#### t_004_insert -- **The generated code**: +1. **The generated code**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, + #[derive(SpacetimeType)] + pub struct Rect { + width: i32, + height: i32, } - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); + #[derive(SpacetimeType)] + pub enum Shape { + Circle(i32), + Rectangle(Rect), } - ``` -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { + #[table(name = result)] + pub struct ResultRow { #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, + id: i32, + value: Shape, } #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); - Ok(()) + fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { + ctx.db.result().insert(ResultRow { + id, + value: Shape::Circle(radius), + }); } ``` -- **The error**: `data_parity_insert_user: spacetime sql failed: no such table: user` - -- **Explain the difference**: - - The generated code didn’t mark struct fields as public, and failed to return a `Result` for the reducer function, which is required by the documentation. - -- **Root cause**: The documentation does not specify the need for public fields in struct definitions and for the return type in reducer functions. - -- **Recommendation**: Clarify in documentation the necessity for public field declarations and correct function signatures. - -#### t_011_helper_function -- **The generated code**: +2. **The golden example**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - id: i32, - sum: i32, - } + use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - fn add(a: i32, b: i32) -> i32 { - a + b + #[derive(SpacetimeType, Clone, Debug)] + pub struct Rect { + pub width: i32, + pub height: i32, } - #[reducer] - fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - let sum = add(a, b); - ctx.db.result().insert(ResultRow { id, sum }); + #[derive(SpacetimeType, Clone, Debug)] + pub enum Shape { + Circle(i32), + Rectangle(Rect), } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(name = result)] pub struct ResultRow { #[primary_key] pub id: i32, - pub sum: i32, + pub value: Shape, } - fn add(a: i32, b: i32) -> i32 { a + b } - #[reducer] - pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); + pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { + ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) }); } ``` -- **The error**: `helper_func_sum_parity: spacetime sql failed: no such table: result` - -- **Explain the difference**: - - Missing public modifiers for struct fields and incorrect reducer function signature. - -- **Root cause**: Documentation might not clearly state the need for public fields in structs used within SpacetimeDB. - -- **Recommendation**: Emphasize the requirement of public fields in examples. - ---- - -### C# / docs Failures (5 total) - -#### 1. t_014_elementary_columns -- **The generated code**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [SpacetimeDB.Table(Name = "Primitive")] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -- **The golden example**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "Primitive")] - public partial struct Primitive - { - [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } +3. **The error**: + ```plaintext + sum_type_row_parity: spacetime sql failed: no such table: `result`. ``` -- **The error**: `no such table: primitive` - -- **Explain the difference**: Field visibility was not explicitly made public in the generated code, which is a requirement for SpacetimeDB. +4. **Explain the difference**: + - The LLM did not make the necessary fields public, which can lead to visibility issues with database inserts. -- **Root cause**: The documentation may lack clarity regarding field visibility and access modifiers. +5. **Root cause**: + - The documentation may not be clear about the visibility of struct fields. -- **Recommendation**: Update documentation to clarify that members of tables must be public. +6. **Recommendation**: + - Update the documentation to stress the importance of public access for all fields that interact with the database. --- -(Continue this format for the remaining C# failures...) +### Summary Recommendations ---- - -### Conclusion - -This comprehensive analysis of SpacetimeDB benchmark test failures highlights key areas where the documentation can improve self-guidance for developers. Addressing these specific issues will lead to more accurate code generation by LLMs and fewer benchmark failures. +- **Visibility**: The documentation should consistently emphasize that public access is necessary for struct fields used in SpacetimeDB. +- **Reducer Necessity**: Clarify when reducers should be defined in accordance with specific use cases (e.g., initial data setup). +- **Attribute Usage**: Provide specific examples of attribute usage, including trait bounds for types like `ScheduleAt` and when/how to annotate functions within data models. + +By addressing the above gaps in the documentation, future errors can be mitigated, making it easier for developers to utilize SpacetimeDB effectively. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index de9b098fefa..02fff94c87c 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 20/27 | 76.4% ⬆️ +2.1% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% | -| Rust | rustdoc_json | **total** | 46/61 | **75.9%** ⬆️ +1.1% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | +| Rust | rustdoc_json | schema | 18/34 | 55.3% ⬇️ -20.0% | +| Rust | rustdoc_json | **total** | 43/61 | **70.6%** ⬇️ -5.3% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | -| C# | docs | schema | 21/34 | 63.7% | -| C# | docs | **total** | 48/61 | **83.5%** ⬆️ +4.5% | +| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 27/27 | 100.0% | +| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | +| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | _Compared against master branch baseline_ -Generated at: 2026-01-26T15:42:12.228Z +Generated at: 2026-01-27T16:47:52.904Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 51adf7934c1..1f525cb5996 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975549166Z", - "finished_at": "2026-01-26T15:31:47.073747240Z" + "started_at": "2026-01-27T16:32:33.607423222Z", + "finished_at": "2026-01-27T16:34:08.828085880Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,25 +69,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.974869540Z", - "finished_at": "2026-01-26T15:31:51.422745193Z" + "started_at": "2026-01-27T16:34:07.172274692Z", + "finished_at": "2026-01-27T16:35:04.564067732Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::info!(\"tick\");\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling constant_time_eq v0.4.2\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling syn v2.0.114\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975688871Z", - "finished_at": "2026-01-26T15:32:30.863567064Z" + "started_at": "2026-01-27T16:34:08.828209290Z", + "finished_at": "2026-01-27T16:35:26.918284244Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::ReducerContext;\nuse spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -125,32 +125,37 @@ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-003-struct-in-table-golden", "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "init()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975098187Z", - "finished_at": "2026-01-26T15:31:55.504039476Z" + "started_at": "2026-01-27T16:34:07.169845549Z", + "finished_at": "2026-01-27T16:35:07.116018289Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &spacetimedb::ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -158,6 +163,25 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { + "data_parity_insert_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:35791" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -166,33 +190,25 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } - }, - "data_parity_insert_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002173ebf56f08f752bb2ca55d0547329e19fb14624413a7089817795d01ab/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975407164Z", - "finished_at": "2026-01-26T15:31:42.241214465Z" + "started_at": "2026-01-27T16:34:08.208097234Z", + "finished_at": "2026-01-27T16:35:13.309948498Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.user().id().update(user);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -216,7 +232,7 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" } }, "seed_users_row": { @@ -234,18 +250,18 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:42.241280124Z", - "finished_at": "2026-01-26T15:32:22.818349314Z" + "started_at": "2026-01-27T16:32:30.244848815Z", + "finished_at": "2026-01-27T16:34:12.499371947Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -269,13 +285,6 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -284,25 +293,32 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:49.227905993Z", - "finished_at": "2026-01-26T15:32:24.290487857Z" + "started_at": "2026-01-27T16:32:20.195987848Z", + "finished_at": "2026-01-27T16:34:08.193093663Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_007_crud", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n if let Some(mut u) = users.id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n users.id().update(u);\n }\n\n users.id().delete(&2);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n let _ = users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n let _ = users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -310,28 +326,65 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:17:11\n |\n17 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | users.try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:24:11\n |\n24 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | users.try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } + }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:35791" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975950122Z", - "finished_at": "2026-01-26T15:32:16.012153929Z" + "started_at": "2026-01-27T16:32:23.303951905Z", + "finished_at": "2026-01-27T16:34:10.621743269Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let _ = ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -359,7 +412,7 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" } }, "schema_parity": { @@ -370,25 +423,25 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.511640110Z", - "finished_at": "2026-01-26T15:32:25.220589761Z" + "started_at": "2026-01-27T16:34:08.199928363Z", + "finished_at": "2026-01-27T16:35:12.331943993Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let _ = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n let _ = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -396,13 +449,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "schema_parity": { @@ -413,36 +475,27 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.073777338Z", - "finished_at": "2026-01-26T15:32:23.492928074Z" + "started_at": "2026-01-27T16:32:30.244175665Z", + "finished_at": "2026-01-27T16:34:05.774125311Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -465,24 +518,24 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975818475Z", - "finished_at": "2026-01-26T15:31:49.227873175Z" + "started_at": "2026-01-27T16:32:22.434743282Z", + "finished_at": "2026-01-27T16:34:04.716053382Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -491,25 +544,22 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "helper_func_sum_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 2, + 3 + ], "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20018816d0c78b41d0a1ebdd62b788e682bdd9fd17555d6ba26e941ae120e74/sql)\n", - "phase": "sql_golden" + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:35791" } }, "helper_func_sum_abs": { @@ -520,21 +570,34 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975246967Z", - "finished_at": "2026-01-26T15:31:47.511599186Z" + "started_at": "2026-01-27T16:34:08.208709446Z", + "finished_at": "2026-01-27T16:35:04.173989578Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -557,7 +620,16 @@ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_score", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -568,34 +640,25 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } - }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:24.290533494Z", - "finished_at": "2026-01-26T15:32:53.440043766Z" + "started_at": "2026-01-27T16:32:19.074381549Z", + "finished_at": "2026-01-27T16:34:04.285926069Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -603,23 +666,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:34081" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -628,11 +674,19 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20002a07d9e6caa3aff0cce8d70357fc0dab19fc7bec72594d9469cfd9e0d34/sql)\n", + "phase": "sql_golden" + } + }, "sum_type_row_count": { "pass": true, "partial": 1.0, @@ -644,18 +698,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:29.610801727Z", - "finished_at": "2026-01-26T15:33:11.012437922Z" + "started_at": "2026-01-27T16:30:59.526974942Z", + "finished_at": "2026-01-27T16:32:33.607356316Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -663,15 +717,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -680,7 +725,7 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } @@ -696,23 +741,32 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:35791" + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:25.220640456Z", - "finished_at": "2026-01-26T15:32:53.137088661Z" + "started_at": "2026-01-27T16:32:12.570231067Z", + "finished_at": "2026-01-27T16:34:07.169774438Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.profile().id().find(1i32).is_none() {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -720,56 +774,50 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_columns_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20022cf10da13cb5686a3d982e8ce97257e0a609cd8cdbf4b75448c0605a05c/sql)\n", + "phase": "sql_golden" } }, - "product_type_columns_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:30.863583165Z", - "finished_at": "2026-01-26T15:33:07.055207496Z" + "started_at": "2026-01-27T16:30:59.529279381Z", + "finished_at": "2026-01-27T16:32:19.074245877Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawing = ctx.db.drawing();\n if drawing.count() == 0 {\n drawing.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawing();\n if drawings.count() == 0 {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -781,18 +829,10 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20084ed1de024b608d5d036674102ffea5602feb56e936989c5737d1fa347f1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200d48d9295216edd01ffee50fc27f7162d3c6ce2ff1442fb683d8707911992/sql)\n", "phase": "sql_golden" } }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c200f0f8cf7ce43e0b888fecfda5572f02aac26af6bea4ac6867cab46bd5e36f/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -801,25 +841,33 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } + }, + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200ed636d1809b5fcb164f6ee86a539d4bdcbe0e1bcc7e6343341fa806bb61a/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:47.540914487Z", - "finished_at": "2026-01-26T15:33:39.147225272Z" + "started_at": "2026-01-27T16:30:59.526533974Z", + "finished_at": "2026-01-27T16:32:27.393830604Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -831,24 +879,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:11\n |\n22 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:23.493005867Z", - "finished_at": "2026-01-26T15:33:38.753749775Z" + "started_at": "2026-01-27T16:30:59.525424836Z", + "finished_at": "2026-01-27T16:32:23.303916011Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n let ids: Vec = account.iter().map(|r| r.id).collect();\n for id in ids {\n account.id().delete(&id);\n }\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -857,17 +905,24 @@ "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c2002a3ca814e8fc4e610af01b0cba5740eac41619685f4959b780669796967a/sql)\n", + "phase": "sql_golden" + } + }, + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:35791", + "tables_diff": null, + "tables_equal": true } }, "constraints_seed_two_rows": { @@ -878,34 +933,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:16.012174316Z", - "finished_at": "2026-01-26T15:32:54.091817951Z" + "started_at": "2026-01-27T16:30:59.520529105Z", + "finished_at": "2026-01-27T16:32:22.433999235Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 5, - "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exact dataset\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().delete(&m);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().delete(&u);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().delete(&g);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -913,69 +955,28 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.3\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling ppv-lite86 v0.2.21\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:33:36\n |\n 33 | ctx.db.membership().delete(&m);\n | ------ ^^ expected `Membership`, found `&Membership`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 33 - ctx.db.membership().delete(&m);\n 33 + ctx.db.membership().delete(m);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:36:30\n |\n 36 | ctx.db.user().delete(&u);\n | ------ ^^ expected `User`, found `&User`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 36 - ctx.db.user().delete(&u);\n 36 + ctx.db.user().delete(u);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:39:31\n |\n 39 | ctx.db.group().delete(&g);\n | ------ ^^ expected `Group`, found `&Group`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 39 - ctx.db.group().delete(&g);\n 39 + ctx.db.group().delete(g);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:22.818452395Z", - "finished_at": "2026-01-26T15:33:12.862647549Z" + "started_at": "2026-01-27T16:30:59.527880905Z", + "finished_at": "2026-01-27T16:32:20.195931017Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(p.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entity_tbl = ctx.db.entity();\n entity_tbl.id().delete(&1);\n entity_tbl.id().delete(&2);\n entity_tbl.insert(Entity { id: 1 });\n entity_tbl.insert(Entity { id: 2 });\n\n // Positions\n let pos_tbl = ctx.db.position();\n pos_tbl.entity_id().delete(&1);\n pos_tbl.entity_id().delete(&2);\n pos_tbl.insert(Position { entity_id: 1, x: 0, y: 0 });\n pos_tbl.insert(Position { entity_id: 2, x: 10, y: 0 });\n\n // Velocities\n let vel_tbl = ctx.db.velocity();\n vel_tbl.entity_id().delete(&1);\n vel_tbl.entity_id().delete(&2);\n vel_tbl.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n vel_tbl.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n let pos_tbl = ctx.db.position();\n let vel_idx = ctx.db.velocity().entity_id();\n let next_tbl = ctx.db.next_position();\n\n for p in pos_tbl.iter() {\n if let Some(v) = vel_idx.find(p.entity_id) {\n let new_x = p.x + v.vx;\n let new_y = p.y + v.vy;\n\n if let Some(mut existing) = next_tbl.entity_id().find(p.entity_id) {\n existing.x = new_x;\n existing.y = new_y;\n let _updated = next_tbl.entity_id().update(existing);\n } else {\n next_tbl.insert(NextPosition {\n entity_id: p.entity_id,\n x: new_x,\n y: new_y,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -983,27 +984,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", - "phase": "sql" - } - }, "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } }, @@ -1015,33 +1008,41 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } }, - "ecs_step_next_positions_count": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:51.422780227Z", - "finished_at": "2026-01-26T15:32:47.540737141Z" + "started_at": "2026-01-27T16:30:59.528443334Z", + "finished_at": "2026-01-27T16:32:30.244044278Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.log();\n\n // Clear existing rows to ensure the table contains exactly the seeded rows\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1049,13 +1050,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, "schema_parity": { @@ -1066,7 +1067,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:35791", "tables_diff": null, "tables_equal": true } @@ -1080,19 +1081,19 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, - "mcindex_lookup_u7_d1": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:55.504076208Z", - "finished_at": "2026-01-26T15:32:29.610771248Z" + "started_at": "2026-01-27T16:30:59.525967187Z", + "finished_at": "2026-01-27T16:32:12.570150592Z" } } } @@ -1114,7 +1115,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1132,8 +1133,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.378312906Z", - "finished_at": "2026-01-26T15:34:06.312973975Z" + "started_at": "2026-01-27T16:37:07.698626526Z", + "finished_at": "2026-01-27T16:38:22.803963062Z" }, "t_001_basic_tables": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1161,8 +1162,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.366156833Z", - "finished_at": "2026-01-26T15:34:21.141281364Z" + "started_at": "2026-01-27T16:37:11.008540155Z", + "finished_at": "2026-01-27T16:38:33.131965123Z" }, "t_002_scheduled_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1172,7 +1173,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) { }\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1190,8 +1191,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.381646420Z", - "finished_at": "2026-01-26T15:35:08.176674329Z" + "started_at": "2026-01-27T16:37:40.463756751Z", + "finished_at": "2026-01-27T16:38:46.674647986Z" }, "t_003_struct_in_table": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1201,7 +1202,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1219,8 +1220,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.368608920Z", - "finished_at": "2026-01-26T15:34:21.997549378Z" + "started_at": "2026-01-27T16:37:10.913641809Z", + "finished_at": "2026-01-27T16:38:28.540002548Z" }, "t_004_insert": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1256,8 +1257,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.375009734Z", - "finished_at": "2026-01-26T15:35:05.975054748Z" + "started_at": "2026-01-27T16:37:12.398406350Z", + "finished_at": "2026-01-27T16:38:29.957852183Z" }, "t_005_update": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1275,20 +1276,20 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "data_parity_update_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" } }, "seed_users_row": { @@ -1302,8 +1303,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:06.313026786Z", - "finished_at": "2026-01-26T15:34:38.434768448Z" + "started_at": "2026-01-27T16:37:05.840143086Z", + "finished_at": "2026-01-27T16:38:27.673339086Z" }, "t_006_delete": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1321,13 +1322,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, + "seed_users_row": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -1338,19 +1339,19 @@ "phase": "describe_golden" } }, - "seed_users_row": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:38.434794482Z", - "finished_at": "2026-01-26T15:35:08.669675470Z" + "started_at": "2026-01-27T16:36:39.493868942Z", + "finished_at": "2026-01-27T16:37:10.913606401Z" }, "t_007_crud": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1360,7 +1361,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1368,21 +1369,13 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id1_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, "notes": { "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -1393,19 +1386,27 @@ "phase": "describe_golden" } }, - "crud_total_count_one": { + "crud_row_id1_parity": { "pass": false, "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.388241938Z", - "finished_at": "2026-01-26T15:35:08.142795838Z" + "started_at": "2026-01-27T16:36:46.682301461Z", + "finished_at": "2026-01-27T16:37:30.046001557Z" }, "t_008_index_lookup": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1415,7 +1416,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let row = ResultRow { id: u.id, name: u.name.clone() };\n if ctx.db.result().id().find(u.id).is_some() {\n ctx.db.result().id().update(row);\n } else {\n ctx.db.result().insert(row);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1450,8 +1451,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.997572287Z", - "finished_at": "2026-01-26T15:34:53.278840948Z" + "started_at": "2026-01-27T16:37:11.012380843Z", + "finished_at": "2026-01-27T16:38:41.409499176Z" }, "t_009_init": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1461,7 +1462,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1478,22 +1479,22 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "init_total_two": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_seed_alice": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -1506,8 +1507,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.141305493Z", - "finished_at": "2026-01-26T15:34:43.189086037Z" + "started_at": "2026-01-27T16:36:57.771441165Z", + "finished_at": "2026-01-27T16:37:40.463630543Z" }, "t_010_connect": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1517,7 +1518,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1535,8 +1536,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.384904874Z", - "finished_at": "2026-01-26T15:35:08.049424018Z" + "started_at": "2026-01-27T16:36:39.717206337Z", + "finished_at": "2026-01-27T16:37:11.008504607Z" }, "t_011_helper_function": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1546,7 +1547,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1581,8 +1582,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.371740140Z", - "finished_at": "2026-01-26T15:35:03.413658925Z" + "started_at": "2026-01-27T16:37:30.046112807Z", + "finished_at": "2026-01-27T16:38:42.478505901Z" }, "t_012_spacetime_product_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1592,7 +1593,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1627,8 +1628,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.176748927Z", - "finished_at": "2026-01-26T15:35:33.331707923Z" + "started_at": "2026-01-27T16:36:36.282196616Z", + "finished_at": "2026-01-27T16:37:12.398375835Z" }, "t_013_spacetime_sum_type": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1638,7 +1639,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1646,6 +1647,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" + } + }, "sum_type_row_count": { "pass": false, "partial": 0.0, @@ -1662,19 +1671,11 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", "phase": "describe_golden" } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.669696395Z", - "finished_at": "2026-01-26T15:35:36.590766489Z" + "started_at": "2026-01-27T16:35:27.761679764Z", + "finished_at": "2026-01-27T16:36:39.717100673Z" }, "t_014_elementary_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1684,7 +1685,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000_i64,\n price: 1.5_f32,\n ratio: 2.25_f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1692,6 +1693,15 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1707,20 +1717,11 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "call_reducer_golden" } - }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.180313713Z", - "finished_at": "2026-01-26T15:35:31.695987649Z" + "started_at": "2026-01-27T16:36:31.329332201Z", + "finished_at": "2026-01-27T16:37:07.698588033Z" }, "t_015_product_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1730,7 +1731,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".into(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".into(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1747,26 +1748,26 @@ "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:17.784622487Z", - "finished_at": "2026-01-26T15:35:56.892022012Z" + "started_at": "2026-01-27T16:35:27.778895055Z", + "finished_at": "2026-01-27T16:36:36.282016589Z" }, "t_016_sum_type_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1776,7 +1777,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1792,26 +1793,26 @@ "phase": "describe_golden" } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c200e332ee39012f68748e8e0058b1df01bfd4bd42fa50c7430f6e03a52a13d7/sql)\n", - "phase": "sql" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c2008da059ad558df05ae36956ea06f6277a53e08f40f6950324728e3d63cfb8/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:20.422565504Z", - "finished_at": "2026-01-26T15:35:53.522136009Z" + "started_at": "2026-01-27T16:35:27.755209342Z", + "finished_at": "2026-01-27T16:37:05.840061596Z" }, "t_017_scheduled_columns": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1821,7 +1822,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let interval = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: interval,\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1829,14 +1830,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } - }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1845,11 +1838,19 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.142877410Z", - "finished_at": "2026-01-26T15:35:53.117891027Z" + "started_at": "2026-01-27T16:35:27.744787479Z", + "finished_at": "2026-01-27T16:37:10.715377293Z" }, "t_018_constraints": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1859,7 +1860,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().id().delete(&1);\n ctx.db.account().id().delete(&2);\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1894,73 +1895,37 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:03.413773673Z", - "finished_at": "2026-01-26T15:35:40.991202532Z" + "started_at": "2026-01-27T16:35:27.741095372Z", + "finished_at": "2026-01-27T16:36:39.493832562Z" }, "t_019_many_to_many": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:05.975134623Z", - "finished_at": "2026-01-26T15:35:55.889702970Z" + "started_at": "2026-01-27T16:38:46.674706560Z", + "finished_at": "2026-01-27T16:38:46.674706560Z" }, "t_020_ecs": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -1970,7 +1935,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n let id1 = 1;\n if ctx.db.entity().id().find(id1).is_none() {\n ctx.db.entity().insert(Entity { id: id1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n let id2 = 2;\n if ctx.db.entity().id().find(id2).is_none() {\n ctx.db.entity().insert(Entity { id: id2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1978,19 +1943,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, @@ -2002,11 +1967,11 @@ "phase": "describe_golden" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } }, @@ -2014,14 +1979,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:43.189115873Z", - "finished_at": "2026-01-26T15:35:20.422539334Z" + "started_at": "2026-01-27T16:35:27.774072550Z", + "finished_at": "2026-01-27T16:36:31.328546056Z" }, "t_021_multi_column_index": { "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", @@ -2031,7 +1996,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2039,23 +2004,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2073,11 +2021,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:53.278868473Z", - "finished_at": "2026-01-26T15:35:17.784590880Z" + "started_at": "2026-01-27T16:35:27.749626445Z", + "finished_at": "2026-01-27T16:36:46.682079929Z" } } } @@ -2298,15 +2263,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:48.351519518Z", - "finished_at": "2026-01-26T15:39:15.916317553Z" + "started_at": "2026-01-27T16:45:33.624143092Z", + "finished_at": "2026-01-27T16:46:27.480022955Z" }, "t_001_basic_tables": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2332,15 +2297,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.611034643Z", - "finished_at": "2026-01-26T15:39:10.630797919Z" + "started_at": "2026-01-27T16:45:58.037479404Z", + "finished_at": "2026-01-27T16:46:46.072177909Z" }, "t_002_scheduled_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2350,7 +2315,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2366,15 +2331,15 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:55.763350829Z", - "finished_at": "2026-01-26T15:39:35.117748962Z" + "started_at": "2026-01-27T16:46:46.072324097Z", + "finished_at": "2026-01-27T16:47:52.804047987Z" }, "t_003_struct_in_table": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2384,7 +2349,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2400,15 +2365,15 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.614826701Z", - "finished_at": "2026-01-26T15:38:48.351485744Z" + "started_at": "2026-01-27T16:45:44.449789708Z", + "finished_at": "2026-01-27T16:46:27.856049534Z" }, "t_004_insert": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2418,7 +2383,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2426,19 +2391,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -2455,13 +2407,26 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.624534968Z", - "finished_at": "2026-01-26T15:38:55.763316545Z" + "started_at": "2026-01-27T16:46:29.820748855Z", + "finished_at": "2026-01-27T16:47:18.121969583Z" }, "t_005_update": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2471,7 +2436,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2487,7 +2452,7 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -2508,7 +2473,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, "seed_users_row": { @@ -2520,8 +2485,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:15.916349271Z", - "finished_at": "2026-01-26T15:39:57.685388275Z" + "started_at": "2026-01-27T16:45:30.312651937Z", + "finished_at": "2026-01-27T16:46:29.809622619Z" }, "t_006_delete": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2531,7 +2496,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2563,15 +2528,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:51.394765047Z", - "finished_at": "2026-01-26T15:40:20.535132716Z" + "started_at": "2026-01-27T16:44:29.592100157Z", + "finished_at": "2026-01-27T16:45:30.312571408Z" }, "t_007_crud": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2581,7 +2546,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n // delete Id=2\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2589,17 +2554,13 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "crud_row_id2_deleted": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } }, "crud_total_count_one": { @@ -2622,22 +2583,26 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, - "crud_row_id2_deleted": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:10.630849129Z", - "finished_at": "2026-01-26T15:39:51.394722435Z" + "started_at": "2026-01-27T16:44:36.788013061Z", + "finished_at": "2026-01-27T16:45:44.449720987Z" }, "t_008_index_lookup": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2647,7 +2612,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Id.Delete(id);\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2655,19 +2620,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -2688,13 +2640,26 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:35.117796353Z", - "finished_at": "2026-01-26T15:40:10.865109962Z" + "started_at": "2026-01-27T16:46:29.809976029Z", + "finished_at": "2026-01-27T16:47:52.369916015Z" }, "t_009_init": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2704,7 +2669,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2712,22 +2677,26 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, "init_seed_alice": { @@ -2739,23 +2708,19 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "schema_parity": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:33.038774271Z", - "finished_at": "2026-01-26T15:40:09.097960805Z" + "started_at": "2026-01-27T16:45:00.485893120Z", + "finished_at": "2026-01-27T16:45:58.037338911Z" }, "t_010_connect": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2765,7 +2730,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2781,15 +2746,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:07.129948444Z", - "finished_at": "2026-01-26T15:39:33.038733830Z" + "started_at": "2026-01-27T16:44:29.599798551Z", + "finished_at": "2026-01-27T16:45:33.624088924Z" }, "t_011_helper_function": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2799,7 +2764,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2815,7 +2780,7 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -2835,7 +2800,7 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, "helper_func_sum_abs": { @@ -2849,8 +2814,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.619502537Z", - "finished_at": "2026-01-26T15:39:07.129901962Z" + "started_at": "2026-01-27T16:46:29.832685293Z", + "finished_at": "2026-01-27T16:47:21.941754653Z" }, "t_012_spacetime_product_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2860,7 +2825,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right },\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2868,17 +2833,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, "product_type_row_parity": { @@ -2896,22 +2857,26 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" } }, - "product_type_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:45.264170257Z", - "finished_at": "2026-01-26T15:41:10.281722543Z" + "started_at": "2026-01-27T16:44:13.602642076Z", + "finished_at": "2026-01-27T16:45:00.485815025Z" }, "t_013_spacetime_sum_type": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2921,7 +2886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2937,20 +2902,11 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, "sum_type_row_parity": { "pass": true, "partial": 1.0, @@ -2965,13 +2921,22 @@ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetCircle", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:32995" + } + }, + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.281769568Z", - "finished_at": "2026-01-26T15:42:04.928512771Z" + "started_at": "2026-01-27T16:42:33.200019323Z", + "finished_at": "2026-01-27T16:43:38.115938270Z" }, "t_014_elementary_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -2981,7 +2946,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -2989,11 +2954,24 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true + } + }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20015ce3d6d6c97d4d92483e7212bbbdc2017cebf40569b0de2758a8c06d8c9/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c2004f34121dd1f4ae79dcdbda66d549feb82a5d63960ba14565d5cc8b739c97/sql)\n", "phase": "sql" } }, @@ -3001,27 +2979,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2002dc67d9e1fd4e2f7be96328c9cadfe424ab52e14114ffe15bf0808847523/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200543789d2e744179bd83fb1415d87abe1ef9050fbc5d65555128998b27490/sql)\n", "phase": "sql_golden" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:59.748869772Z", - "finished_at": "2026-01-26T15:41:30.294907546Z" + "started_at": "2026-01-27T16:43:38.116010007Z", + "finished_at": "2026-01-27T16:44:36.787949888Z" }, "t_015_product_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3031,7 +2996,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3048,37 +3013,37 @@ "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:32995" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.284990504Z", - "finished_at": "2026-01-26T15:41:51.914201320Z" + "started_at": "2026-01-27T16:43:18.813807927Z", + "finished_at": "2026-01-27T16:44:27.109398198Z" }, "t_016_sum_type_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3088,7 +3053,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3096,6 +3061,14 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200a6ce7b1295896bce91792742020e2d963fc68bfca6d8ff9804106ed485f8/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3104,31 +3077,23 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2007921ce30371e511ce337c700f912b2dc1842029ec51095bcee487a1fb5a2/sql)\n", - "phase": "sql_golden" - } - }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20008b060dbbbe2ff941855e14b04bb0c88b907fca414a2d82874c574b1cf60/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200bbb059f1df73468f61a994e99b02f150ebdc9dce5396fc2cc064f5ebb917/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:12.893136011Z", - "finished_at": "2026-01-26T15:42:12.178253367Z" + "started_at": "2026-01-27T16:41:37.626585140Z", + "finished_at": "2026-01-27T16:43:18.813740398Z" }, "t_017_scheduled_columns": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3138,7 +3103,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled tick; no-op\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3146,14 +3111,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20077e1b6738482c8dd0108623bedd0c70f80fb40fb09fda1f12bbe2c6972bf/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3162,15 +3119,23 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200283fa6a6ce49e85887cd3c05f6cd55c39e71c5cdd5dc78a67d45df2c41c4/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:27.963020126Z", - "finished_at": "2026-01-26T15:41:12.893111985Z" + "started_at": "2026-01-27T16:41:37.619157633Z", + "finished_at": "2026-01-27T16:42:40.765400962Z" }, "t_018_constraints": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3180,7 +3145,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Account.Iter())\n {\n ctx.Db.Account.Id.Delete(row.Id);\n }\n\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -3188,6 +3153,20 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "constraints_row_parity_after_seed": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-018-constraints-golden", + "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:32995" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3196,7 +3175,7 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3209,25 +3188,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-018-constraints-golden", - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:41793" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:10.865255262Z", - "finished_at": "2026-01-26T15:40:59.748828990Z" + "started_at": "2026-01-27T16:41:37.615926753Z", + "finished_at": "2026-01-27T16:42:33.199931492Z" }, "t_019_many_to_many": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3237,7 +3202,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3253,7 +3218,7 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3267,15 +3232,6 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - }, "m2m_has_1_10": { "pass": true, "partial": 1.0, @@ -3293,11 +3249,20 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:20.535170845Z", - "finished_at": "2026-01-26T15:41:10.109404307Z" + "started_at": "2026-01-27T16:42:40.765467116Z", + "finished_at": "2026-01-27T16:44:13.602547935Z" }, "t_020_ecs": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", @@ -3307,7 +3272,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Insert positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Insert velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(p.EntityId) is Velocity v)\n {\n int nx = p.X + v.VX;\n int ny = p.Y + v.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(p.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = p.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3315,27 +3280,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } }, @@ -3347,7 +3304,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:32995", "tables_diff": null, "tables_equal": true } @@ -3356,24 +3313,32 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:57.685434816Z", - "finished_at": "2026-01-26T15:40:45.264108157Z" + "started_at": "2026-01-27T16:43:06.400065322Z", + "finished_at": "2026-01-27T16:44:29.591963094Z" }, "t_021_multi_column_index": { "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Log\", Public = true)]\n [Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3381,18 +3346,50 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" + } + }, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:32995", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:09.098001666Z", - "finished_at": "2026-01-26T15:40:27.962996760Z" + "started_at": "2026-01-27T16:41:37.622718575Z", + "finished_at": "2026-01-27T16:43:06.399960112Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 4e4aa267103..56cb6e89647 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-26T15:42:12.228Z", + "generated_at": "2026-01-27T16:47:52.904Z", "by_language": { "csharp": { "modes": { @@ -20,19 +20,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 21, - "pass_pct": 61.764706, - "task_pass_equiv": 6.3666663, - "task_pass_pct": 63.666664 + "passed_tests": 25, + "pass_pct": 73.52941, + "task_pass_equiv": 7.3666663, + "task_pass_pct": 73.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 18.366667, - "task_pass_pct": 83.48485 + "passed_tests": 52, + "pass_pct": 85.2459, + "task_pass_equiv": 19.366667, + "task_pass_pct": 88.030304 } } } @@ -56,53 +56,53 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 30, + "passed_tests": 4, + "pass_pct": 13.333333, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 57, + "passed_tests": 9, + "pass_pct": 15.789474, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 20, - "pass_pct": 74.07407, - "task_pass_equiv": 9.166667, - "task_pass_pct": 76.38889 + "passed_tests": 25, + "pass_pct": 92.59259, + "task_pass_equiv": 10.0, + "task_pass_pct": 83.33333 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.5333333, - "task_pass_pct": 75.333336 + "passed_tests": 18, + "pass_pct": 52.941177, + "task_pass_equiv": 5.5333333, + "task_pass_pct": 55.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 16.7, - "task_pass_pct": 75.909096 + "passed_tests": 43, + "pass_pct": 70.491806, + "task_pass_equiv": 15.533334, + "task_pass_pct": 70.606064 } } } diff --git a/licenses/BSL.txt b/licenses/BSL.txt index c2dcfcf84cb..74636fe1f48 100644 --- a/licenses/BSL.txt +++ b/licenses/BSL.txt @@ -5,7 +5,7 @@ Business Source License 1.1 Parameters Licensor: Clockwork Laboratories, Inc. -Licensed Work: SpacetimeDB 1.11.3 +Licensed Work: SpacetimeDB 1.12.0 The Licensed Work is (c) 2023 Clockwork Laboratories, Inc. @@ -21,7 +21,7 @@ Additional Use Grant: You may make use of the Licensed Work provided your Licensed Work by creating tables whose schemas are controlled by such third parties. -Change Date: 2031-01-14 +Change Date: 2031-01-24 Change License: GNU Affero General Public License v3.0 with a linking exception diff --git a/sdks/csharp/SpacetimeDB.ClientSDK.csproj b/sdks/csharp/SpacetimeDB.ClientSDK.csproj index 540213c769d..f2416ae875f 100644 --- a/sdks/csharp/SpacetimeDB.ClientSDK.csproj +++ b/sdks/csharp/SpacetimeDB.ClientSDK.csproj @@ -16,8 +16,8 @@ logo.png README.md https://github.com/clockworklabs/com.clockworklabs.spacetimedbsdk - 1.11.2 - 1.11.2 + 1.12.0 + 1.12.0 $(DefaultItemExcludes);*~/** packages @@ -25,7 +25,7 @@ - + diff --git a/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj b/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj index 6259cd0f3b9..d0056e5c9f4 100644 --- a/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj +++ b/sdks/csharp/examples~/regression-tests/server/StdbModule.csproj @@ -8,7 +8,7 @@ - + diff --git a/sdks/csharp/package.json b/sdks/csharp/package.json index 22d2fcd7b82..7493b06b217 100644 --- a/sdks/csharp/package.json +++ b/sdks/csharp/package.json @@ -1,7 +1,7 @@ { "name": "com.clockworklabs.spacetimedbsdk", "displayName": "SpacetimeDB SDK", - "version": "1.11.2", + "version": "1.12.0", "description": "The SpacetimeDB Client SDK is a software development kit (SDK) designed to interact with and manipulate SpacetimeDB modules..", "keywords": [], "author": { diff --git a/templates/basic-cs/spacetimedb/StdbModule.csproj b/templates/basic-cs/spacetimedb/StdbModule.csproj index 6259cd0f3b9..d0056e5c9f4 100644 --- a/templates/basic-cs/spacetimedb/StdbModule.csproj +++ b/templates/basic-cs/spacetimedb/StdbModule.csproj @@ -8,7 +8,7 @@ - + diff --git a/templates/basic-rs/Cargo.toml b/templates/basic-rs/Cargo.toml index 5d1fabdc702..fe1aca10515 100644 --- a/templates/basic-rs/Cargo.toml +++ b/templates/basic-rs/Cargo.toml @@ -4,4 +4,4 @@ version = "0.1.0" edition = "2021" [dependencies] -spacetimedb-sdk = "1.11.*" +spacetimedb-sdk = "1.12.*" diff --git a/templates/basic-rs/spacetimedb/Cargo.toml b/templates/basic-rs/spacetimedb/Cargo.toml index 271b883365e..67ba62bad82 100644 --- a/templates/basic-rs/spacetimedb/Cargo.toml +++ b/templates/basic-rs/spacetimedb/Cargo.toml @@ -9,5 +9,5 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -spacetimedb = "1.11.*" +spacetimedb = "1.12.*" log = "0.4" diff --git a/templates/chat-console-cs/spacetimedb/StdbModule.csproj b/templates/chat-console-cs/spacetimedb/StdbModule.csproj index fa9208f2da5..0e47bfed155 100644 --- a/templates/chat-console-cs/spacetimedb/StdbModule.csproj +++ b/templates/chat-console-cs/spacetimedb/StdbModule.csproj @@ -14,7 +14,7 @@ - + diff --git a/templates/chat-react-ts/src/module_bindings/index.ts b/templates/chat-react-ts/src/module_bindings/index.ts index 5dc82db57f6..8d00c314360 100644 --- a/templates/chat-react-ts/src/module_bindings/index.ts +++ b/templates/chat-react-ts/src/module_bindings/index.ts @@ -1,7 +1,7 @@ // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. -// This was generated using spacetimedb cli version 1.11.3 (commit f9bca6a8df856d950360b40cbce744fcbffc9a63). +// This was generated using spacetimedb cli version 1.12.0 (commit 4a6228fec73c9fefa0ee32ceebf06c91933453ab). /* eslint-disable */ /* tslint:disable */ @@ -107,7 +107,7 @@ const proceduresSchema = __procedures(); /** The remote SpacetimeDB module schema, both runtime and type information. */ const REMOTE_MODULE = { versionInfo: { - cliVersion: '1.11.3' as const, + cliVersion: '1.12.0' as const, }, tables: tablesSchema.schemaType.tables, reducers: reducersSchema.reducersType.reducers,