diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 44f3a564731..17b5c26d78c 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,20 +4,21 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 35 +- **Total failures analyzed**: 36 --- -# Analysis of SpacetimeDB Benchmark Failures +# SpacetimeDB Benchmark Failures Analysis -Here's an overview of the SpacetimeDB benchmark failures organized by programming language and mode, providing actionable insights for documentation updates. +This document analyzes test failures in the SpacetimeDB benchmark organized by language and mode. For each failure, we provide the generated code, the expected code, the error message, and a detailed explanation along with actionable recommendations. -## Rust / rustdoc_json Failures (9 Total) +## Rust / rustdoc_json Failures (8 total) -### Compile/Publish Errors (3 failures) +### Compile/Publish Errors (2 failures) -#### t_002_scheduled_table -- **Generated Code**: +#### t_002_scheduled_table & t_017_scheduled_columns + +1. **The generated code**: ```rust use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; @@ -28,8 +29,22 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin scheduled_id: u64, scheduled_at: ScheduleAt, } + + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)), + }); + } + } + + #[reducer] + pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {} ``` -- **Golden Example**: + +2. **The expected code**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -41,99 +56,55 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub scheduled_id: u64, pub scheduled_at: ScheduleAt, } - ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: The LLM used `schedule(reducer = tick, column = scheduled_at)` instead of `scheduled(tick)`. -- **Root Cause**: The reducer scheduling syntax was incorrectly formatted. -- **Recommendation**: Update the documentation to clarify the correct syntax for scheduled attributes. - ---- - -#### t_015_product_type_columns -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - #[derive(SpacetimeType)] - pub struct Address { - street: String, - zip: i32, - } - ``` -- **Golden Example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + #[reducer] + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} - #[derive(SpacetimeType, Clone, Debug)] - pub struct Address { - pub street: String, - pub zip: i32, + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); } ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Missing `pub` modifiers on struct fields and missing `Clone` and `Debug` traits. -- **Root Cause**: The documentation didn't clearly indicate the necessary visibility modifiers and traits for SpacetimeDB types. -- **Recommendation**: Enhance examples in the documentation to include field visibility and common traits for struct definitions. ---- +3. **The error**: `publish_error: spacetime publish failed (exit=1)` -#### t_017_scheduled_columns -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; +4. **Explain the difference**: + - Incorrect `#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]` should use `#[table(name = tick_timer, scheduled(tick))]` + - Use of `ScheduleAt::repeat` is incorrect; it should use a proper time duration constructor. - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - ``` -- **Golden Example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; +5. **Root cause**: The documentation lacks clarity on scheduling syntax and constructors for time intervals in scheduled tasks. - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Same scheduling error as in `t_002_scheduled_table`. -- **Root Cause**: Similar to above, incorrect usage of scheduling syntax. -- **Recommendation**: Provide explicit examples illustrating scheduled and unscheduled table definitions. +6. **Recommendation**: Update documentation to emphasize using `scheduled(tick)` and correct constructors for `ScheduleAt` using `Duration::from_millis`. --- ### Other Failures (6 failures) -#### t_003_struct_in_table -- **Generated Code**: +#### t_003_struct_in_table, t_004_insert, t_007_crud, t_011_helper_function, t_016_sum_type_columns + +1. **The generated code** (e.g., for `t_003`): ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; + use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType}; - #[derive(SpacetimeType)] + #[derive(SpacetimeType, Clone)] pub struct Position { - x: i32, - y: i32, + pub x: i32, + pub y: i32, } - #[table(name = entity)] + #[spacetimedb::table(name = entity)] pub struct Entity { #[primary_key] - id: i32, - pos: Position, + pub id: i32, + pub pos: Position, } - - #[reducer(init)] - pub fn init(_ctx: &ReducerContext) {} ``` -- **Golden Example**: + +2. **The expected code**: ```rust use spacetimedb::{table, SpacetimeType}; @@ -142,6 +113,7 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub x: i32, pub y: i32, } + #[table(name = entity)] pub struct Entity { #[primary_key] @@ -149,227 +121,134 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub pos: Position, } ``` -- **Error**: `schema_parity: reducers differ - expected [], got ["add_entity()", "init()"]` -- **Difference**: Missing `pub` modifiers led to visibility issues; `add_entity` reducer was not included. -- **Root Cause**: The documentation should indicate proper visibility for struct members. -- **Recommendation**: Include visibility guidance in struct and reducer examples to prevent visibility mismatches. + +3. **The error**: `schema_parity: reducers differ - expected [], got [...]` + +4. **Explain the difference**: Missing `pub` for fields in structs which are not public, causing access issues. + +5. **Root cause**: Insufficient detail in documentation about struct visibility and reducing/scheduling attributes. + +6. **Recommendation**: Clarify that public fields are required for structs defining database tables. --- -#### t_004_insert -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; +#### Additional Observations: - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } +- The focus must be on both visibility modifiers and correct API signatures for reducers and tables. +- The need for `Result<(), String>` in reducer functions is missing in many generated snippets. - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); - } - ``` -- **Golden Example**: +--- + +## Rust / docs Failures (22 total) + +### Other Failures (22 failures) + +#### t_000_empty_reducers, t_001_basic_tables, t_002_scheduled_table, t_004_insert + +1. **The generated code** (for `t_000_empty_reducers`): ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::ReducerContext; - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } + #[spacetimedb::reducer] + pub fn empty_reducer_no_args(_ctx: &ReducerContext) {} + ``` + +2. **The expected code**: + ```rust + use spacetimedb::{reducer, ReducerContext}; #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); + pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { Ok(()) } ``` -- **Error**: `data_parity_insert_user: spacetime sql failed: Error: no such table: user` -- **Difference**: Missing `pub` modifiers on struct fields and the reducer function lacked error handling. -- **Root Cause**: The need for visibility on struct fields and proper error handling for database operations is not emphasized in documentation. -- **Recommendation**: Emphasize that all struct fields must be public and demonstrate typical error handling in reducers. + +3. **The error**: `schema_parity: describe failed: WARNING: This command is UNSTABLE` + +4. **Explain the difference**: Missing return type `Result<(), String>` for all reducer functions causes the failure. + +5. **Root cause**: Documentation does not clearly specify that all reducer functions must return a `Result`. + +6. **Recommendation**: Update the documentation to explicitly require a `Result` return type for all reducer functions to avoid compilation errors. --- -## C# / docs Failures (4 Total) +## C# / docs Failures (6 total) + +### Other Failures (6 failures) -### Other Failures (4 failures) +#### t_008_index_lookup, t_013_spacetime_sum_type -#### t_014_elementary_columns -- **Generated Code**: +1. **The generated code** (for `t_008_index_lookup`): ```csharp using SpacetimeDB; public static partial class Module { - [SpacetimeDB.Table(Name = "Primitive", Public = true)] - public partial struct Primitive + [SpacetimeDB.Table(Name = "User")] + public partial struct User { [SpacetimeDB.PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; public string Name; + public int Age; + public bool Active; } [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) + public static void LookupUserName(ReducerContext ctx, int id) { - ctx.Db.Primitive.Insert(new Primitive + var user = ctx.Db.User.Id.Find(id); + if (user != null) { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); + ctx.Db.Result.Insert(new Result + { + Id = user.Id, + Name = user.Name + }); + } } } ``` -- **Golden Example**: + +2. **The expected code**: ```csharp using SpacetimeDB; public static partial class Module { - [Table(Name = "Primitive")] - public partial struct Primitive + [Table(Name = "User")] + public partial struct User { [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; public string Name; + public int Age; + public bool Active; } [Reducer] - public static void Seed(ReducerContext ctx) + public static void LookupUserName(ReducerContext ctx, int id) { - ctx.Db.Primitive.Insert(new Primitive { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); + var u = ctx.Db.User.Id.Find(id); + if (u.HasValue) + { + var row = u.Value; + ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name }); + } } } ``` -- **Error**: `spacetime sql failed: Error: no such table: primitive` -- **Difference**: The `Public` attribute is used incorrectly, which may lead to schema mismatch or missing construct. -- **Root Cause**: The correct usage of attributes may not be clearly noted in the documentation. -- **Recommendation**: Validate the appropriate usage of attributes in table and struct definitions. ---- +3. **The error**: `publish_error: spacetime build (csharp) failed (exit=1)` -#### t_016_sum_type_columns -- **Generated Code**: - ```csharp - using SpacetimeDB; +4. **Explain the difference**: Use of `user != null` instead of checking `u.HasValue`, which is necessary for nullable types. - public static partial class Module - { - [SpacetimeDB.Table(Name = "Drawing", Public = true)] - public partial struct Drawing - { - [SpacetimeDB.PrimaryKey] - public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... - } - ``` -- **Golden Example**: - ```csharp - using SpacetimeDB; +5. **Root cause**: Lacking examples for nullable types or option types in the given context. - public static partial class Module - { - [Table(Name = "Drawing")] - public partial struct Drawing - { - [PrimaryKey] public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... - } - ``` -- **Error**: `spacetime sql failed: Error: no such table: drawings` -- **Difference**: Similar to the previous error, the `Public` attribute on a table may not be necessary and may lead to runtime issues. -- **Root Cause**: Documentation isn't explicit about when to use visibility modifiers and how they interact with SpacetimeDB configuration. -- **Recommendation**: Clarify the usage of attribute visibility and provide clear examples. +6. **Recommendation**: Address nullable type usage in the documentation, emphasizing how to correctly check for value presence. --- -#### t_017_scheduled_columns -- **Generated Code**: - ```csharp - using System; - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] - public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } - - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer _timer) { } - - // ... init reducer ... - } - ``` -- **Golden Example**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } - - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer schedule) { } - - // ... init reducer with more clarity ... - } - ``` -- **Error**: `spacetime sql failed: Error: no such table: tick_timer` -- **Difference**: Incorrect usage of parameter names in the reducer; `_timer` should just use a clear name. -- **Root Cause**: The documentation should illustrate effective naming conventions for function parameters. -- **Recommendation**: Include best practices for naming and structuring reducers alongside examples of scheduled fields. - ---- +### Final Thoughts -This analytical report outlines the specific failures within SpacetimeDB benchmarks, pinpointing concrete documentation enhancements to improve understanding and usability. By implementing these changes, developers can reduce occurrence and severity of future benchmark failures. +A thorough review of generator patterns and failure analysis indicates that clarifying visibility, return types, syntax for scheduling, and handling nullable types are crucial improvements for development efficiency and error avoidance in SpacetimeDB. Documenting common patterns and providing clear guidelines will enhance user experience and reduce test failures. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index e6e73436a0f..6a2bcda9a11 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 22/27 | 74.3% ⬇️ -9.0% | -| Rust | rustdoc_json | schema | 23/34 | 65.3% | -| Rust | rustdoc_json | **total** | 45/61 | **70.2%** ⬇️ -4.9% | +| Rust | rustdoc_json | basics | 22/27 | 74.3% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | +| Rust | rustdoc_json | **total** | 48/61 | **74.8%** ⬆️ +4.5% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +5.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +2.3% | -| C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | -| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | +| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 24/27 | 91.7% ⬇️ -8.3% | +| C# | docs | schema | 22/34 | 63.7% ⬇️ -10.0% | +| C# | docs | **total** | 46/61 | **78.9%** ⬇️ -9.1% | _Compared against master branch baseline_ -Generated at: 2026-01-23T12:08:22.290Z +Generated at: 2026-01-23T20:54:53.246Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index b6074d5fccf..69aaf946ba3 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -19,7 +19,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595950645Z", - "finished_at": "2026-01-23T11:56:59.057829079Z" + "started_at": "2026-01-23T20:45:58.319507359Z", + "finished_at": "2026-01-23T20:46:21.669318285Z" }, "t_001_basic_tables": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -53,7 +53,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,15 +69,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595256337Z", - "finished_at": "2026-01-23T11:57:01.112482005Z" + "started_at": "2026-01-23T20:45:58.318770844Z", + "finished_at": "2026-01-23T20:46:19.785596995Z" }, "t_002_scheduled_table": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -87,7 +87,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let t = ctx.db.tick_timer();\n for row in t.iter() {\n t.scheduled_id().delete(&row.scheduled_id);\n }\n t.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,14 +99,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.16\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling bytemuck v1.24.0\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling second-stack v0.3.5\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:14\n |\n18 | t.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:20\n |\n14 | let t = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:35\n |\n20 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling zmij v1.0.16\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:44\n |\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596087756Z", - "finished_at": "2026-01-23T11:57:51.917718856Z" + "started_at": "2026-01-23T20:45:58.319658080Z", + "finished_at": "2026-01-23T20:46:56.319486952Z" }, "t_003_struct_in_table": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -116,7 +116,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn insert_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}\n\n#[spacetimedb::reducer]\npub fn set_pos(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n if let Some(mut e) = ctx.db.entity().id().find(id) {\n e.pos.x = x;\n e.pos.y = y;\n ctx.db.entity().id().update(e);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -133,20 +133,21 @@ "reducers_diff": { "only_golden": [], "only_llm": [ - "add_entity()", - "init()" + "init()", + "insert_entity()", + "set_pos()" ] }, "reducers_equal": false, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595509989Z", - "finished_at": "2026-01-23T11:57:06.869986441Z" + "started_at": "2026-01-23T20:45:58.319022832Z", + "finished_at": "2026-01-23T20:46:29.838664307Z" }, "t_004_insert": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -172,7 +173,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -181,14 +182,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2008f3693ac65719c539bc7c4992fe451a1bec99bb058273f581f5ee525695b/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200e9fa53e3a843d1820d588423033ed683af3525a68a4dc3f8578dabd3b437/sql)\n", "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595811655Z", - "finished_at": "2026-01-23T11:57:05.537662622Z" + "started_at": "2026-01-23T20:45:58.319345443Z", + "finished_at": "2026-01-23T20:46:19.313225077Z" }, "t_005_update": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -198,7 +199,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let new_row = User { id, name, age, active };\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(new_row);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -206,17 +207,23 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:34477" } }, "seed_users_row": { @@ -226,29 +233,23 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:43035" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:59.057861578Z", - "finished_at": "2026-01-23T11:57:53.866612446Z" + "started_at": "2026-01-23T20:46:19.313268675Z", + "finished_at": "2026-01-23T20:46:50.997884926Z" }, "t_006_delete": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -258,7 +259,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -266,6 +267,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -274,18 +282,11 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "delete_user_count_zero": { "pass": true, "partial": 1.0, @@ -297,8 +298,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840911142Z", - "finished_at": "2026-01-23T11:57:48.533997072Z" + "started_at": "2026-01-23T20:46:21.669348863Z", + "finished_at": "2026-01-23T20:46:43.139558965Z" }, "t_007_crud": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -308,7 +309,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let _ = users.id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n\n users.id().delete(&2);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n let id_to_delete = 2;\n users.id().delete(&id_to_delete);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -325,6 +326,14 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, + "crud_row_id1_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200fecea2f4fd1e842ce9bf311fc7af775ed2f1c09d17c59d34a450538b65ed/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -333,7 +342,7 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -346,19 +355,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM user" } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2009e8af0d449c17ae3da6d300c97be156e1eb9f1da87b7c6b9cefbcde67475/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596351347Z", - "finished_at": "2026-01-23T11:57:10.126845149Z" + "started_at": "2026-01-23T20:45:58.319932266Z", + "finished_at": "2026-01-23T20:46:41.862776693Z" }, "t_008_index_lookup": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -368,7 +369,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\nfn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -376,13 +377,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -391,11 +385,18 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -409,13 +410,13 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840644958Z", - "finished_at": "2026-01-23T11:57:40.599601481Z" + "started_at": "2026-01-23T20:46:20.508322641Z", + "finished_at": "2026-01-23T20:46:56.770679153Z" }, "t_009_init": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -425,7 +426,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -433,6 +434,15 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { + "init_seed_alice": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -441,42 +451,33 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:01.112523418Z", - "finished_at": "2026-01-23T11:57:24.446822222Z" + "started_at": "2026-01-23T20:46:19.785638677Z", + "finished_at": "2026-01-23T20:46:42.338746730Z" }, "t_010_connect": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -502,15 +503,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596220949Z", - "finished_at": "2026-01-23T11:57:02.704314900Z" + "started_at": "2026-01-23T20:45:58.319798861Z", + "finished_at": "2026-01-23T20:46:24.449136396Z" }, "t_011_helper_function": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -528,14 +529,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c20092cdd1c98eaef99ecfeb113a11c7cee6251c76ce243f1d56372113126f59/sql)\n", - "phase": "sql_golden" - } - }, "helper_func_sum_abs": { "pass": true, "partial": 1.0, @@ -545,6 +538,14 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, + "helper_func_sum_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200a76ada80a129ab1d98b1132742dd63568f85071327226e59b33240f90ed8/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -553,15 +554,15 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595665553Z", - "finished_at": "2026-01-23T11:57:02.840609277Z" + "started_at": "2026-01-23T20:45:58.319183557Z", + "finished_at": "2026-01-23T20:46:20.508270684Z" }, "t_012_spacetime_product_type": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -579,6 +580,19 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, "product_type_row_count": { "pass": true, "partial": 1.0, @@ -603,26 +617,13 @@ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_score", - "server": "http://127.0.0.1:43035" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:40.599657560Z", - "finished_at": "2026-01-23T11:58:09.609768604Z" + "started_at": "2026-01-23T20:46:50.997932201Z", + "finished_at": "2026-01-23T20:47:18.590168533Z" }, "t_013_spacetime_sum_type": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -632,7 +633,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -640,23 +641,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:43035" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -665,7 +649,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -678,11 +662,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_circle", + "server": "http://127.0.0.1:34477" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:48.534039773Z", - "finished_at": "2026-01-23T11:58:41.485665974Z" + "started_at": "2026-01-23T20:46:56.770721316Z", + "finished_at": "2026-01-23T20:47:22.846786446Z" }, "t_014_elementary_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -692,7 +693,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -711,7 +712,16 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:34477" + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } }, "schema_parity": { @@ -722,34 +732,25 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:47.891667065Z", - "finished_at": "2026-01-23T11:58:20.627171437Z" + "started_at": "2026-01-23T20:46:56.319506066Z", + "finished_at": "2026-01-23T20:47:21.058802558Z" }, "t_015_product_type_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.profile();\n for row in tbl.iter().collect::>() {\n tbl.delete(&row);\n }\n tbl.insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -757,18 +758,46 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling hex v0.4.3\n Compiling cc v1.2.54\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:29:20\n |\n 29 | tbl.delete(&row);\n | ------ ^^^^ expected `Profile`, found `&Profile`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 29 - tbl.delete(&row);\n 29 + tbl.delete(row);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "query": "SELECT id, home, work, pos FROM profile WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:34477" + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917727311Z", - "finished_at": "2026-01-23T11:58:32.335767995Z" + "started_at": "2026-01-23T20:46:56.770961297Z", + "finished_at": "2026-01-23T20:47:32.576819834Z" }, "t_016_sum_type_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -778,7 +807,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let row = Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n };\n ctx.db.drawing().insert(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let a = Shape::Circle(10);\n let b = Shape::Rectangle(Rect { width: 4, height: 6 });\n ctx.db.drawing().insert(Drawing { id: 1, a, b });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -786,14 +815,6 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200fa0a7bdaa0b3fe32198acc7d7c51970587917bc93ce14e7d0e54f859979b/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -802,7 +823,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -811,14 +832,22 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200b5079878ebf16faffc0bc609578eb070b7c5eb623e6f0ec9ca7c1e73e040/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002a23ef16d310345e1d4ce4b5d48569b614371f4967976a5819022bbc2662/sql)\n", "phase": "sql" } + }, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c20040bc998fdc9ab3d5a21e438444dc0653e63e1691252800cd646cfc6722b3/sql)\n", + "phase": "sql_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917991193Z", - "finished_at": "2026-01-23T11:58:22.938115353Z" + "started_at": "2026-01-23T20:47:00.920013549Z", + "finished_at": "2026-01-23T20:47:31.508612717Z" }, "t_017_scheduled_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -828,7 +857,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n // Scheduled tick; no-op for this task.\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -840,14 +869,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling cc v1.2.54\n Compiling itoa v1.0.17\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:20:36\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:15\n |\n19 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:20:16\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:22:39\n |\n22 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:31.320145539Z", - "finished_at": "2026-01-23T11:58:26.729342562Z" + "started_at": "2026-01-23T20:46:43.139606842Z", + "finished_at": "2026-01-23T20:47:30.076456163Z" }, "t_018_constraints": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -857,7 +886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n tbl.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n for row in tbl.iter() {\n tbl.id().delete(&row.id);\n }\n\n tbl.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n tbl.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -865,6 +894,19 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -885,26 +927,13 @@ "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT id, email, name FROM account WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:43035" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:10.126874202Z", - "finished_at": "2026-01-23T11:57:47.891616379Z" + "started_at": "2026-01-23T20:46:41.862819773Z", + "finished_at": "2026-01-23T20:47:07.104266828Z" }, "t_019_many_to_many": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -914,7 +943,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&row.id);\n }\n for row in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&row.user_id);\n }\n for row in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&row.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -940,15 +969,6 @@ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, "m2m_has_1_20": { "pass": true, "partial": 1.0, @@ -966,15 +986,24 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:24.446860971Z", - "finished_at": "2026-01-23T11:58:08.134821703Z" + "started_at": "2026-01-23T20:46:42.338778389Z", + "finished_at": "2026-01-23T20:47:21.898328928Z" }, "t_020_ecs": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -984,7 +1013,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entities = ctx.db.entity();\n if entities.id().find(1).is_none() {\n entities.insert(Entity { id: 1 });\n }\n if entities.id().find(2).is_none() {\n entities.insert(Entity { id: 2 });\n }\n\n // Positions\n let positions = ctx.db.position();\n if let Some(mut p) = positions.entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut p) = positions.entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n\n // Velocities\n let velocities = ctx.db.velocity();\n if let Some(mut v) = velocities.entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n if let Some(mut v) = velocities.entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n let next_positions = ctx.db.next_position();\n\n for pos in positions.iter() {\n if let Some(vel) = velocities.entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut np) = next_positions.entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n let _ = next_positions.entity_id().update(np);\n } else {\n next_positions.insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -992,19 +1021,11 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } }, @@ -1016,7 +1037,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -1025,22 +1046,30 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:05.537695688Z", - "finished_at": "2026-01-23T11:57:51.917639786Z" + "started_at": "2026-01-23T20:46:24.449168444Z", + "finished_at": "2026-01-23T20:47:00.919947661Z" }, "t_021_multi_column_index": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -1050,7 +1079,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1066,7 +1095,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -1100,8 +1129,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:06.870008743Z", - "finished_at": "2026-01-23T11:57:31.320114801Z" + "started_at": "2026-01-23T20:46:29.838705690Z", + "finished_at": "2026-01-23T20:46:56.594909460Z" } } } @@ -1109,21 +1138,21 @@ }, { "mode": "docs", - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1141,11 +1170,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.716223450Z", - "finished_at": "2026-01-23T11:59:49.748708946Z" + "started_at": "2026-01-23T20:47:32.846152158Z", + "finished_at": "2026-01-23T20:48:46.489399789Z" }, "t_001_basic_tables": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, @@ -1170,18 +1199,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.704113020Z", - "finished_at": "2026-01-23T11:59:42.429041793Z" + "started_at": "2026-01-23T20:47:32.833609179Z", + "finished_at": "2026-01-23T20:48:40.565099319Z" }, "t_002_scheduled_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n let desired = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n let count = ctx.db.tick_timer().count();\n if count == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: desired,\n });\n } else {\n let mut iter = ctx.db.tick_timer().iter();\n if let Some(mut first) = iter.next() {\n first.scheduled_at = desired;\n ctx.db.tick_timer().scheduled_id().update(first);\n }\n for extra in iter {\n ctx.db.tick_timer().scheduled_id().delete(&extra.scheduled_id);\n }\n }\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1199,18 +1228,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.719410738Z", - "finished_at": "2026-01-23T11:59:57.789054932Z" + "started_at": "2026-01-23T20:47:32.849575285Z", + "finished_at": "2026-01-23T20:48:55.497529150Z" }, "t_003_struct_in_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1228,11 +1257,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.706682369Z", - "finished_at": "2026-01-23T11:59:26.096561811Z" + "started_at": "2026-01-23T20:47:32.836252652Z", + "finished_at": "2026-01-23T20:48:05.833400632Z" }, "t_004_insert": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1265,18 +1294,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.712966773Z", - "finished_at": "2026-01-23T11:59:51.858461717Z" + "started_at": "2026-01-23T20:47:32.842787873Z", + "finished_at": "2026-01-23T20:48:29.038416446Z" }, "t_005_update": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1284,12 +1313,12 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_update_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" } }, "seed_users_row": { @@ -1301,21 +1330,21 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "data_parity_update_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:26.096589408Z", - "finished_at": "2026-01-23T11:59:53.555166464Z" + "started_at": "2026-01-23T20:48:05.833493117Z", + "finished_at": "2026-01-23T20:48:31.203040278Z" }, "t_006_delete": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1330,6 +1359,14 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" + } + }, "seed_users_row": { "pass": false, "partial": 0.0, @@ -1347,22 +1384,14 @@ "expected": 0, "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:43.249955612Z", - "finished_at": "2026-01-23T12:00:06.215097718Z" + "started_at": "2026-01-23T20:48:31.203078568Z", + "finished_at": "2026-01-23T20:48:50.906207073Z" }, "t_007_crud": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_007_crud", "lang": "rust", "golden_published": true, @@ -1377,6 +1406,14 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" + } + }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -1386,12 +1423,13 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "crud_row_id1_parity": { @@ -1401,30 +1439,21 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", "phase": "call_reducer_golden" } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.725929053Z", - "finished_at": "2026-01-23T11:59:43.249835351Z" + "started_at": "2026-01-23T20:47:32.856090158Z", + "finished_at": "2026-01-23T20:48:19.458976831Z" }, "t_008_index_lookup": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name.clone() });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1459,18 +1488,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:42.429072085Z", - "finished_at": "2026-01-23T12:00:06.978561278Z" + "started_at": "2026-01-23T20:48:29.038498151Z", + "finished_at": "2026-01-23T20:48:56.042485955Z" }, "t_009_init": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1478,13 +1507,21 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", + "phase": "describe_golden" + } + }, + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "init_seed_alice": { @@ -1496,37 +1533,29 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", - "phase": "describe_golden" - } - }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:40.767658962Z", - "finished_at": "2026-01-23T12:00:00.250974348Z" + "started_at": "2026-01-23T20:48:19.459032083Z", + "finished_at": "2026-01-23T20:48:44.457914124Z" }, "t_010_connect": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_010_connect", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1544,11 +1573,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.722667055Z", - "finished_at": "2026-01-23T11:59:44.003413383Z" + "started_at": "2026-01-23T20:47:32.852844068Z", + "finished_at": "2026-01-23T20:48:51.666864219Z" }, "t_011_helper_function": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, @@ -1563,15 +1592,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1587,21 +1607,30 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", "phase": "call_reducer_golden" } + }, + "helper_func_sum_abs": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.709720362Z", - "finished_at": "2026-01-23T11:59:40.767569416Z" + "started_at": "2026-01-23T20:47:32.839497474Z", + "finished_at": "2026-01-23T20:48:51.151329754Z" }, "t_012_spacetime_product_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n let value = Score { left, right };\n ctx.db.result().insert(ResultRow { id, value });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1609,20 +1638,20 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "call_reducer_golden" } }, - "product_type_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } }, "product_type_row_count": { @@ -1636,18 +1665,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:00.251008970Z", - "finished_at": "2026-01-23T12:00:21.554582472Z" + "started_at": "2026-01-23T20:48:51.666947087Z", + "finished_at": "2026-01-23T20:49:14.419221949Z" }, "t_013_spacetime_sum_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1655,6 +1684,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "describe_golden" + } + }, "sum_type_row_count": { "pass": false, "partial": 0.0, @@ -1671,29 +1708,21 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", "phase": "call_reducer_golden" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:06.978584823Z", - "finished_at": "2026-01-23T12:00:33.966822730Z" + "started_at": "2026-01-23T20:48:56.042516286Z", + "finished_at": "2026-01-23T20:49:19.391193690Z" }, "t_014_elementary_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1701,14 +1730,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, @@ -1725,21 +1746,29 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "describe_golden" } + }, + "elementary_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:06.215125322Z", - "finished_at": "2026-01-23T12:00:33.902324216Z" + "started_at": "2026-01-23T20:48:55.497603714Z", + "finished_at": "2026-01-23T20:49:21.590560437Z" }, "t_015_product_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1774,18 +1803,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:18.383198165Z", - "finished_at": "2026-01-23T12:01:34.378780677Z" + "started_at": "2026-01-23T20:49:09.915973191Z", + "finished_at": "2026-01-23T20:49:35.664676341Z" }, "t_016_sum_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1793,12 +1822,12 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200de67ac503f1e38be9c09680d3df61e49eb3a7d66ef3c6c8c371fb66d4edf/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -1809,28 +1838,28 @@ "phase": "describe_golden" } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c200b73d487f28a58ba0c1daee3f8e73c716071a8ecb6ca5512534eaaf3f94f4/sql)\n", - "phase": "sql" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:21.554609433Z", - "finished_at": "2026-01-23T12:00:50.848757403Z" + "started_at": "2026-01-23T20:49:10.455837761Z", + "finished_at": "2026-01-23T20:49:49.430676143Z" }, "t_017_scheduled_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1857,18 +1886,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:57.789164943Z", - "finished_at": "2026-01-23T12:00:37.720475314Z" + "started_at": "2026-01-23T20:48:51.151415261Z", + "finished_at": "2026-01-23T20:49:22.096623158Z" }, "t_018_constraints": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|a| a.id).collect();\n for id in ids {\n ctx.db.account().id().delete(id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1903,83 +1932,47 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:51.858540280Z", - "finished_at": "2026-01-23T12:00:18.383168342Z" + "started_at": "2026-01-23T20:48:46.489485052Z", + "finished_at": "2026-01-23T20:49:14.344881271Z" }, "t_019_many_to_many": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Ensure exactly these rows exist by clearing existing state.\n // Delete memberships first to avoid FK-like logical dependencies.\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User {\n user_id: 1,\n name: \"Alice\".to_string(),\n });\n ctx.db.user().insert(User {\n user_id: 2,\n name: \"Bob\".to_string(),\n });\n\n // Insert groups\n ctx.db.group().insert(Group {\n group_id: 10,\n title: \"Admin\".to_string(),\n });\n ctx.db.group().insert(Group {\n group_id: 20,\n title: \"Dev\".to_string(),\n });\n\n // Insert memberships\n ctx.db.membership().insert(Membership {\n id: 1,\n user_id: 1,\n group_id: 10,\n });\n ctx.db.membership().insert(Membership {\n id: 2,\n user_id: 1,\n group_id: 20,\n });\n ctx.db.membership().insert(Membership {\n id: 3,\n user_id: 2,\n group_id: 20,\n });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" + "error": "POST https://api.openai.com/v1/responses -> 502 Bad Gateway: \r\n502 Bad Gateway\r\n\r\n

502 Bad Gateway

\r\n
cloudflare
\r\n\r\n\r\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:53.555195009Z", - "finished_at": "2026-01-23T12:00:37.548228808Z" + "started_at": "2026-01-23T20:49:49.431161113Z", + "finished_at": "2026-01-23T20:49:49.431161113Z" }, "t_020_ecs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 1, x: 0, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 1, vx: 1, vy: 0 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 2, x: 10, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 2, vx: -2, vy: 3 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let row = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(row);\n } else {\n ctx.db.next_position().insert(row);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1995,52 +1988,52 @@ "phase": "describe_golden" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:44.003507320Z", - "finished_at": "2026-01-23T12:00:45.914588769Z" + "started_at": "2026-01-23T20:48:40.565127071Z", + "finished_at": "2026-01-23T20:49:16.253635323Z" }, "t_021_multi_column_index": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.log().iter() {\n ctx.db.log().id().delete(row.id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2048,15 +2041,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -2066,6 +2050,14 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2075,18 +2067,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:49.748795415Z", - "finished_at": "2026-01-23T12:00:23.844832103Z" + "started_at": "2026-01-23T20:48:44.457949164Z", + "finished_at": "2026-01-23T20:49:10.455801685Z" } } } @@ -2277,14 +2270,14 @@ "modes": [ { "mode": "docs", - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2307,25 +2300,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:18.656745307Z", - "finished_at": "2026-01-23T12:04:50.054718472Z" + "started_at": "2026-01-23T20:51:58.516007671Z", + "finished_at": "2026-01-23T20:52:19.852644961Z" }, "t_001_basic_tables": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2341,25 +2334,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.882300132Z", - "finished_at": "2026-01-23T12:04:36.458730908Z" + "started_at": "2026-01-23T20:51:08.718544439Z", + "finished_at": "2026-01-23T20:51:58.515971496Z" }, "t_002_scheduled_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_002_scheduled_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer tickTimer)\n {\n // Scheduled reducer invoked by TickTimer; no-op by specification.\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2375,18 +2368,18 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:24.844091061Z", - "finished_at": "2026-01-23T12:05:24.876865213Z" + "started_at": "2026-01-23T20:52:02.545010135Z", + "finished_at": "2026-01-23T20:52:47.129183717Z" }, "t_003_struct_in_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, @@ -2409,25 +2402,25 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.885907238Z", - "finished_at": "2026-01-23T12:04:30.187542427Z" + "started_at": "2026-01-23T20:51:08.721057407Z", + "finished_at": "2026-01-23T20:52:05.996428958Z" }, "t_004_insert": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_004_insert", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2443,7 +2436,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } @@ -2464,23 +2457,23 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.897716646Z", - "finished_at": "2026-01-23T12:04:18.656638441Z" + "started_at": "2026-01-23T20:51:08.727492764Z", + "finished_at": "2026-01-23T20:52:08.554796558Z" }, "t_005_update": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing == null)\n {\n throw new System.Exception(\"User not found\");\n }\n\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n ctx.Db.User.Id.Update(updated);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n\n if (ctx.Db.User.Id.Find(id) != null)\n {\n ctx.Db.User.Id.Update(updated);\n }\n else\n {\n ctx.Db.User.Insert(updated);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2488,6 +2481,19 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } + }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2504,7 +2510,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "seed_users_row": { @@ -2513,34 +2519,21 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:50.054760665Z", - "finished_at": "2026-01-23T12:05:47.466386003Z" + "started_at": "2026-01-23T20:52:19.852673232Z", + "finished_at": "2026-01-23T20:52:53.389808270Z" }, "t_006_delete": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2548,20 +2541,20 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { + "delete_user_count_zero": { "pass": true, "partial": 1.0, "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" } }, - "delete_user_count_zero": { + "seed_users_row": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -2572,25 +2565,25 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.880862690Z", - "finished_at": "2026-01-23T12:06:10.897631772Z" + "started_at": "2026-01-23T20:52:51.107488271Z", + "finished_at": "2026-01-23T20:53:13.731066286Z" }, "t_007_crud": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2598,28 +2591,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, "crud_row_id1_parity": { "pass": true, "partial": 1.0, @@ -2631,7 +2602,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "crud_total_count_one": { @@ -2642,21 +2613,43 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM User" } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:36.458767943Z", - "finished_at": "2026-01-23T12:05:24.618417126Z" + "started_at": "2026-01-23T20:52:08.554828637Z", + "finished_at": "2026-01-23T20:52:51.107450441Z" }, "t_008_index_lookup": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n if (ctx.Db.Result.Id.Find(id) is Result existing)\n {\n existing.Name = user.Name;\n ctx.Db.Result.Id.Update(existing);\n }\n else\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2664,56 +2657,28 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "LookupUserName", - "server": "http://127.0.0.1:35813" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(32,27): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(33,29): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.876913581Z", - "finished_at": "2026-01-23T12:06:14.796358950Z" + "started_at": "2026-01-23T20:52:47.129209751Z", + "finished_at": "2026-01-23T20:53:08.577581818Z" }, "t_009_init": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"User\")]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2729,18 +2694,18 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, "init_total_two": { @@ -2752,29 +2717,29 @@ "sql": "SELECT COUNT(*) AS n FROM User" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:18.162783655Z", - "finished_at": "2026-01-23T12:06:11.635711073Z" + "started_at": "2026-01-23T20:52:36.884011764Z", + "finished_at": "2026-01-23T20:53:00.062192214Z" }, "t_010_connect": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2790,25 +2755,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:30.187575150Z", - "finished_at": "2026-01-23T12:05:18.162739983Z" + "started_at": "2026-01-23T20:52:05.996450500Z", + "finished_at": "2026-01-23T20:52:36.883985146Z" }, "t_011_helper_function": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2816,6 +2781,15 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2824,20 +2798,11 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, "helper_func_sum_parity": { "pass": true, "partial": 1.0, @@ -2853,23 +2818,23 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.890278760Z", - "finished_at": "2026-01-23T12:04:24.844053542Z" + "started_at": "2026-01-23T20:51:08.724127650Z", + "finished_at": "2026-01-23T20:52:02.544972253Z" }, "t_012_spacetime_product_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2877,22 +2842,17 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetScore", - "server": "http://127.0.0.1:35813" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } }, "product_type_row_count": { @@ -2904,33 +2864,38 @@ "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, - "schema_parity": { + "product_type_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 2, + 3 + ], "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetScore", + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:44.805001437Z", - "finished_at": "2026-01-23T12:07:16.896251109Z" + "started_at": "2026-01-23T20:53:40.420411949Z", + "finished_at": "2026-01-23T20:54:07.766554713Z" }, "t_013_spacetime_sum_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_013_spacetime_sum_type", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2938,52 +2903,21 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:35813" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.896327053Z", - "finished_at": "2026-01-23T12:08:07.936339455Z" + "started_at": "2026-01-23T20:54:05.268678013Z", + "finished_at": "2026-01-23T20:54:43.846818259Z" }, "t_014_elementary_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, @@ -2998,49 +2932,49 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2003883ecec05b675876359540c06439933bae20ff47c4c334ae2cfa397c954/sql)\n", + "phase": "sql" } }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20099d4196f0168d509841ccbf52a41e879a5de183a360f652419198a45fa09/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200ba25315ff7445dc543e2e42a2309cd493cfe1974181f53e50fa06c464134/sql)\n", "phase": "sql_golden" } }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20009038c3974b6dd9b27cd833b5bf2548c34fdbd6898aac40238455b2a0d9b/sql)\n", - "phase": "sql" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:54.360546232Z", - "finished_at": "2026-01-23T12:07:34.648602066Z" + "started_at": "2026-01-23T20:53:43.824629255Z", + "finished_at": "2026-01-23T20:54:14.326210872Z" }, "t_015_product_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3048,20 +2982,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:35813" - } - }, "product_type_columns_row_count": { "pass": true, "partial": 1.0, @@ -3079,25 +2999,39 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } + }, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:46871" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.899960622Z", - "finished_at": "2026-01-23T12:07:53.777092694Z" + "started_at": "2026-01-23T20:54:07.846975920Z", + "finished_at": "2026-01-23T20:54:41.128204875Z" }, "t_016_sum_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing { Id = 1, A = a, B = b });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3105,14 +3039,6 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200b0343bd80e3f58914be7679e885cb40098290c8573df5d5e85bf5f1d8191/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3121,33 +3047,41 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2002b630c1898fdca0d5df967295e9965d354ea89c64878bb9c71421ac210de/sql)\n", + "phase": "sql_golden" + } + }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200ad3cc29488a8fe57b5ab9ac0432417ab7d585d48e65d407028387050d856/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200255f75652e874ebf03622fd9c2dfb3b65b42f575d4fa5a95cc36e4182f25/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:31.832777595Z", - "finished_at": "2026-01-23T12:08:22.217518757Z" + "started_at": "2026-01-23T20:54:07.849642740Z", + "finished_at": "2026-01-23T20:54:53.203074499Z" }, "t_017_scheduled_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(50_000 * 10))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer row)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(500_000))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3155,14 +3089,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200805f92c3324ddd49c14c46faf1bfa5b52a3ebae1e3664df720fbd9d4a487/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3171,18 +3097,26 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200bcc4292b7995634fb91d48b6baa5c9096e7f85c4b73e7db1b4f1f377a348/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:40.441707268Z", - "finished_at": "2026-01-23T12:07:31.832707547Z" + "started_at": "2026-01-23T20:53:28.130925948Z", + "finished_at": "2026-01-23T20:54:07.846947117Z" }, "t_018_constraints": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3197,19 +3131,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3221,7 +3142,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "constraints_seed_two_rows": { @@ -3232,21 +3153,34 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.639612541Z", - "finished_at": "2026-01-23T12:06:44.804960222Z" + "started_at": "2026-01-23T20:53:08.577602895Z", + "finished_at": "2026-01-23T20:53:40.420363798Z" }, "t_019_many_to_many": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3254,13 +3188,17 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "m2m_has_1_10": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } }, "memberships_three_rows": { @@ -3272,26 +3210,22 @@ "sql": "SELECT COUNT(*) AS n FROM Membership" } }, - "schema_parity": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, "m2m_has_1_20": { @@ -3305,18 +3239,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:14.796478395Z", - "finished_at": "2026-01-23T12:07:15.676945464Z" + "started_at": "2026-01-23T20:53:13.731095349Z", + "finished_at": "2026-01-23T20:54:05.268633334Z" }, "t_020_ecs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_020_ecs", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n if (ctx.Db.Entity.Id.Find(1) is not Entity e1)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n if (ctx.Db.Position.EntityId.Find(1) is not Position p1)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(1) is not Velocity v1)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n // Entity 2\n if (ctx.Db.Entity.Id.Find(2) is not Entity e2)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n if (ctx.Db.Position.EntityId.Find(2) is not Position p2)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(2) is not Velocity v2)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is not Velocity vel)\n {\n continue;\n }\n\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.Value.VX;\n int nextY = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var np = existing.Value;\n np.X = nextX;\n np.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3324,19 +3258,11 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } }, @@ -3348,41 +3274,49 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", + "phase": "sql" + } + }, "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:47.466492129Z", - "finished_at": "2026-01-23T12:06:40.441631392Z" + "started_at": "2026-01-23T20:52:53.389850738Z", + "finished_at": "2026-01-23T20:53:43.824579798Z" }, "t_021_multi_column_index": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3407,33 +3341,33 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "mcindex_lookup_u7_d2": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.635827651Z", - "finished_at": "2026-01-23T12:06:54.360432909Z" + "started_at": "2026-01-23T20:53:00.062219949Z", + "finished_at": "2026-01-23T20:53:28.130894765Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index f5405f61c51..8fd73afcfd8 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,38 +1,38 @@ { "version": 1, - "generated_at": "2026-01-23T12:08:22.290Z", + "generated_at": "2026-01-23T20:54:53.246Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 + "passed_tests": 24, + "pass_pct": 88.888885, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.3666663, - "task_pass_pct": 73.666664 + "passed_tests": 22, + "pass_pct": 64.70588, + "task_pass_equiv": 6.366667, + "task_pass_pct": 63.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 19.366667, - "task_pass_pct": 88.030304 + "passed_tests": 46, + "pass_pct": 75.409836, + "task_pass_equiv": 17.366667, + "task_pass_pct": 78.9394 } } } @@ -42,7 +42,7 @@ "rust": { "modes": { "docs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "models": { "GPT-5": { "categories": { @@ -56,20 +56,20 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 30, + "passed_tests": 4, + "pass_pct": 13.333333, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 57, + "passed_tests": 9, + "pass_pct": 15.789474, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } @@ -90,19 +90,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.533333, - "task_pass_pct": 65.33333 + "passed_tests": 26, + "pass_pct": 76.47059, + "task_pass_equiv": 7.5333333, + "task_pass_pct": 75.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 45, - "pass_pct": 73.77049, - "task_pass_equiv": 15.45, - "task_pass_pct": 70.22727 + "passed_tests": 48, + "pass_pct": 78.68852, + "task_pass_equiv": 16.45, + "task_pass_pct": 74.772736 } } }